Deploy simready-assets run 20261019-9f72902 (GPU variant) (part 2)

pipeline/bake.py

@@ -186,6 +186,18 @@ def _image_names(cfg: Config) -> Dict[str, str]:
     return names
 
 
+#: A repo pinned at more revisions than this gets ONE rule instead of one per
+#: revision (url_mapping_rules). Above every repo on the submission lists of
+#: 2026-10-15 (most: 19), so their rules are unchanged.
+COLLAPSE_REVISIONS_ABOVE = 32
+
+#: USDSEARCH_URL_MAPPING_RULES is one environment string in the Space image.
+#: Linux refuses any single env or argv string over 128 KiB (MAX_ARG_STRLEN),
+#: so past it every process in the container fails to start; stop the bake
+#: well before that.
+URL_MAPPING_RULES_MAX_BYTES = 100_000
+
+
 def url_mapping_rules(hf_repos: List[List[str]], asset_counts: Optional[List[List]] = None) -> List[Dict]:
     """deepsearch-api's USDSEARCH_URL_MAPPING_RULES for the HF repos a run
     indexed (manifest["hf_repos"]: host, repo_type, repo_id, revision): each
@@ -203,20 +215,44 @@ def url_mapping_rules(hf_repos: List[List[str]], asset_counts: Optional[List[Lis
     the revision with the most packages (`asset_counts`, manifest
     ["hf_repo_asset_counts"]: [host, repo_type, repo_id, revision, n]), the
     one most legacy links point into; without counts, the first listed.
+
+    A repo pinned at more than COLLAPSE_REVISIONS_ABOVE revisions (one upload
+    commit per package: 657 for PalatialSim on 2026-10-15) gets one rule,
+    `hf://.../resolve/` -> `.../blob/`, which carries every revision through
+    unchanged, so the rules stop growing with the pins. It has no
+    `.../blob/main/` alias: search_utils refuses a public prefix that is a
+    prefix of another, and `.../blob/` is one of `.../blob/main/`.
     """
     counts = {tuple(row[:4]): row[4] for row in asset_counts or []}
     alias_revision: Dict[tuple, str] = {}
+    pinned: Dict[tuple, int] = {}
+    unpinned = set()
     for host, repo_type, repo_id, revision in hf_repos:
+        repo = (host, repo_type, repo_id)
         if not hf_addressing.looks_like_sha(revision):
+            unpinned.add(repo)
             continue
-        repo = (host, repo_type, repo_id)
+        pinned[repo] = pinned.get(repo, 0) + 1
         best = alias_revision.get(repo)
         if best is None or counts.get((*repo, revision), 0) > counts.get((*repo, best), 0):
             alias_revision[repo] = revision
+    collapsed = {repo for repo, n in pinned.items() if n > COLLAPSE_REVISIONS_ABOVE and repo not in unpinned}
+    emitted = set()
     rules = []
     for host, repo_type, repo_id, revision in hf_repos:
+        repo = (host, repo_type, repo_id)
         prefix = hf_addressing.REPO_TYPE_PREFIX.get(repo_type, "")
         web = f"https://{host}/" + "/".join(p for p in (prefix, repo_id) if p)
+        if repo in collapsed:
+            if repo in emitted:
+                continue
+            emitted.add(repo)
+            rules.append({
+                "internal": hf_addressing.build(host=host, repo_type=repo_type, repo_id=repo_id,
+                                                revision="", is_folder=True) + "resolve/",
+                "public": f"{web}/blob/",
+            })
+            continue
         rule = {
             "internal": hf_addressing.build(host=host, repo_type=repo_type, repo_id=repo_id,
                                             revision=revision, is_folder=True),
@@ -264,8 +300,14 @@ def hf_space_build_args(cfg: Config, manifest: Dict) -> Dict[str, str]:
     # go stale when the corpus is re-pinned. As a Space Variable it is both
     # the build-arg and the runtime env var. Off for an s3:// corpus.
     rules = url_mapping_rules(manifest.get("hf_repos") or [], manifest.get("hf_repo_asset_counts")) if storage_backend == "storage_api" else []
+    rules_json = json.dumps(rules, separators=(",", ":"))
+    if len(rules_json.encode()) > URL_MAPPING_RULES_MAX_BYTES:
+        raise ValueError(
+            f"USDSEARCH_URL_MAPPING_RULES would be {len(rules_json.encode())} bytes ({len(rules)} rules), over "
+            f"{URL_MAPPING_RULES_MAX_BYTES}: Linux refuses an env string past 128 KiB, so no process in the Space "
+            f"would start. Lower COLLAPSE_REVISIONS_ABOVE or ask the submitter to pin fewer commits.")
     args["USDSEARCH_URL_MAPPING_MODE"] = "rules" if rules else "off"
-    args["USDSEARCH_URL_MAPPING_RULES"] = json.dumps(rules, separators=(",", ":"))
+    args["USDSEARCH_URL_MAPPING_RULES"] = rules_json
     # S3_STORAGE_* is only ever meaningful for an S3-sourced corpus, and
     # threading a source bucket name through for an HF-sourced one would be
     # actively misleading.
@@ -463,14 +505,20 @@ def public_url_smoke(base_url: str, public_prefixes: List[str], timeout_s: float
     is not enough. The stale hand-set rule emitted
     `https://.../blob/main/resolve/<sha>/...` (a 404), which passes it, so
     each URL must start with one of `public_prefixes`
-    (url_mapping_rules()'s `public`s). Returns (ok, detail). Polls like
-    download_smoke(): the seed restores the index after /health is up."""
+    (url_mapping_rules()'s `public`s). Under a collapsed repo's `.../blob/`
+    the next segment must be a commit sha, or that stale form would pass.
+    Returns (ok, detail). Polls like download_smoke(): the seed restores the
+    index after /health is up."""
+    import re
     import time
     import httpx
 
+    def under(u):
+        return any(u.startswith(p) and (not p.endswith("/blob/") or re.match(r"[0-9a-f]{40}/", u[len(p):]))
+                   for p in public_prefixes)
+
     def bad(urls):
-        return [u for u in urls if not (isinstance(u, str) and u.startswith("https://")
-                                        and u.startswith(tuple(public_prefixes)))]
+        return [u for u in urls if not (isinstance(u, str) and u.startswith("https://") and under(u))]
 
     detail = "no URLs returned"
     deadline = time.monotonic() + timeout_s

pipeline/categories.py

@@ -7,8 +7,10 @@ between `wait` and `export`, every indexed asset gets one more USD property, `ce
 
   * a partner (OEM) package — any dataset owner but NVIDIA's own — gets `oem:<owner>`, the HF dataset owner its
     simready-central submission list names (`submissions/<owner>/<dataset>`), never vendor-typed metadata;
+  * a package pinned in config/hierarchy/category_pins.yaml (by package folder) gets its pin, even over its
+    Q-code: a Q-code that covers unlike objects (one "machine" for a safety shutoff and a brake tester), or
+    a package S3 already holds in another category folder;
   * any other asset gets its `simready_metadata_qcode` value(s);
-  * else a pin from config/hierarchy/category_pins.yaml (by package folder);
   * else the one Q-code its prims carry as a semantic label (`semantic:wikidata_qcode:params:semanticData`);
   * else nothing (it stays in the tree's Uncategorized bucket; the stage lists it).
 
@@ -17,6 +19,11 @@ config/hierarchy/vendors.yaml) and a leaf per pin (`augment_hierarchy`), so OEM
 OEM > vendor and the NVIDIA categories stay NVIDIA's. Search, query parsing and every other filter keep reading
 `simready_metadata_qcode`; only the tree reads `central:category`.
 
+Beside the tree goes `install_categories.json` (`install_list`): every package folder of the corpus's HF datasets
+with the category folder the Space's tree files it under, for installers that copy packages into category folders
+(Michael Mallon's S3 copy). A package maps only when all its values sit in the tree under ONE folder path; anything
+else is `needs_review` with a reason, never a guessed folder. The Space serves it as /install_categories.json.
+
     python -m pipeline.run_pipeline categories --env-file ... [--dry-run]
     python -m pipeline.categories hierarchy --env-file ... --from-run <run> --run-id <run> [--dry-run]
 """
@@ -24,7 +31,10 @@ OEM > vendor and the NVIDIA categories stay NVIDIA's. Search, query parsing and
 from __future__ import annotations
 
 import argparse
+import datetime as dt
 import json
+import os
+import re
 import sys
 from collections import Counter
 from pathlib import Path
@@ -44,6 +54,10 @@ VENDORS_FILE = REPO_ROOT / "config" / "hierarchy" / "vendors.yaml"
 PINS_FILE = REPO_ROOT / "config" / "hierarchy" / "category_pins.yaml"
 REPORT = BUILD_DIR / "categories.json"
 HIERARCHY_FILE = "category_hierarchy.json"
+INSTALL_FILE = "install_categories.json"
+INSTALL_SCHEMA = "1.1.0"
+LAST_COMMITS_ENV = "INSTALL_LIST_LAST_COMMITS"  # "off": no HF listing (no last commits, no not-indexed rows)
+QCODE_RE = re.compile(r"^Q[1-9]\d*$")
 
 
 # ── config ───────────────────────────────────────────────────────────────────
@@ -123,36 +137,41 @@ def category_values(url: str, props: Iterable[Dict], vendors: Dict, pins: Dict[s
     owner, package = split_url(url)
     if owner and owner.lower() not in vendors["own"]:
         return [f"{OEM_PREFIX}{owner}"], "oem"
+    if package and package in pins:
+        return [pins[package]["key"]], "pin"
     qcodes = prop_values(props, QCODE_PROPERTY)
     if qcodes:
         return qcodes, "qcode"
-    if package and package in pins:
-        return [pins[package]["key"]], "pin"
     semantic = prop_values(props, SEMANTIC_QCODE_PROPERTY)
     if len(semantic) == 1:
         return semantic, "semantic"
     return [], "none"
 
 
-def with_category(props: List[Dict], values: List[str]) -> List[Dict]:
-    """`props` without any `central:category` entry, plus one entry per value, shaped like the props' own string
-    entries (name/value/value_type and their `_sayt` twins where the index has them)."""
-    kept = [p for p in props or [] if not (isinstance(p, dict) and p.get("name") == PROPERTY)]
+def with_property(props: List[Dict], name: str, values: List[str]) -> List[Dict]:
+    """`props` without any `name` entry, plus one entry per value, shaped like the props' own string entries
+    (name/value/value_type and their `_sayt` twins where the index has them)."""
+    kept = [p for p in props or [] if not (isinstance(p, dict) and p.get("name") == name)]
     template = next((p for p in kept if isinstance(p, dict) and p.get("value_type") == "string"), None)
     keys = set(template) if template else {"name", "value", "value_type", "name_sayt", "value_sayt"}
     out = list(kept)
     for v in values:
-        entry = {"name": PROPERTY, "value": v}
+        entry = {"name": name, "value": v}
         if "value_type" in keys:
             entry["value_type"] = "string"
         if "name_sayt" in keys:
-            entry["name_sayt"] = PROPERTY
+            entry["name_sayt"] = name
         if "value_sayt" in keys:
             entry["value_sayt"] = v
         out.append(entry)
     return out
 
 
+def with_category(props: List[Dict], values: List[str]) -> List[Dict]:
+    """`props` with its `central:category` entries replaced by `values`."""
+    return with_property(props, PROPERTY, values)
+
+
 def _folder_steps(path: str) -> List[Dict]:
     parts = [p for p in path.split("/") if p]
     return [{"qcode": FOLDER_PREFIX + "/".join(parts[: i + 1]), "label": parts[i]} for i in range(len(parts))]
@@ -188,6 +207,202 @@ def augment_hierarchy(doc: Dict, values: Iterable[str], vendors: Dict, pins: Dic
     return out, unplaced
 
 
+# ── install list: package folder -> category folder ──────────────────────────
+
+
+def package_address(url: str) -> Optional[Tuple[str, str, str]]:
+    """(repo id, revision, top-level package folder) of an hf:// address or its public https blob/resolve URL."""
+    if url.startswith("hf://"):
+        try:
+            addr = hf_addressing.parse(url)
+        except hf_addressing.AddressError:
+            return None
+        folder = addr.path.split("/")[0]
+        return (addr.repo_id, addr.revision or "", folder) if folder else None
+    parts = url.split("/")
+    if len(parts) > 9 and parts[3] == "datasets" and parts[6] in ("blob", "resolve") and parts[8]:
+        return f"{parts[4]}/{parts[5]}", parts[7], parts[8]
+    return None
+
+
+def tree_folder(rec: Dict) -> Optional[str]:
+    """The folder a tree entry files its assets under: the labels of its `folder:` steps, plus the vendor for an
+    OEM entry (OEM > LG Electronics). None when the entry has no folder."""
+    steps = rec.get("path") or []
+    labels = [str(s.get("label") or "") for s in steps if str(s.get("qcode") or "").startswith(FOLDER_PREFIX)]
+    if steps and str(steps[-1].get("qcode") or "").startswith(OEM_PREFIX):
+        labels.append(str(steps[-1].get("label") or ""))
+    return "/".join(labels) if labels else None
+
+
+def safe_path(path: str) -> bool:
+    """Every segment a plain folder name: not empty, not . or .., no backslash or control character."""
+    return all(seg.strip() and seg not in (".", "..") and "\\" not in seg and not any(ord(ch) < 32 for ch in seg)
+               for seg in path.split("/"))
+
+
+def _last_commit(raw) -> Optional[Dict]:
+    if not raw:
+        return None
+    if isinstance(raw, dict):
+        oid, title, date = raw.get("id") or raw.get("oid"), raw.get("title"), raw.get("date")
+    else:
+        oid, title, date = getattr(raw, "oid", None), getattr(raw, "title", None), getattr(raw, "date", None)
+    if isinstance(date, dt.datetime):
+        date = date.astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z")
+    return {"id": oid, "date": date, "title": title} if oid else None
+
+
+def package_listings(repos: Iterable[Tuple[str, str]], api=None) -> Tuple[Dict[Tuple[str, str], Dict], str]:
+    """The top-level folders of each (repo id, revision) with their last commit at that revision: one
+    non-recursive expanded tree listing each (Hugging Face pages expanded listings 50 at a time). Fail-soft: what
+    it could list, and a note saying what it could not."""
+    from huggingface_hub import RepoFolder
+
+    if api is None:
+        from .enumerate_assets import _hf_api
+        api = _hf_api()
+    out: Dict[Tuple[str, str], Dict] = {}
+    failed: List[str] = []
+    for repo, rev in sorted(set(repos)):
+        try:
+            entries = api.list_repo_tree(repo, recursive=False, revision=rev or None, repo_type="dataset", expand=True)
+            out[(repo, rev)] = {e.path: _last_commit(getattr(e, "last_commit", None))
+                                for e in entries if isinstance(e, RepoFolder) and not e.path.startswith(".")}
+        except Exception as exc:  # noqa: BLE001 — the list ships without last commits rather than not at all
+            failed.append(f"{repo}@{rev[:8]}: {type(exc).__name__}: {str(exc)[:160]}")
+    return out, ("listed" if not failed else "not listed for " + "; ".join(failed))
+
+
+def install_list(assets: Iterable[Dict], tree: Dict, *, own: Iterable[str], listings: Optional[Dict] = None,
+                 run_id: str = "", corpus: str = "", last_commits: str = "", generated_at: str = "") -> Dict:
+    """`install_categories.json`: every package folder -> the category folder of the Space's tree.
+
+    `assets` are the categories stage's per-file rows ({url, values, rule}); `tree` the augmented
+    category_hierarchy.json; `listings` the HF top-level folders per (repo, revision) from `package_listings`
+    (their last commits, and the folders of our own datasets that no indexed file covers: `not_indexed`).
+    The primary repository (our own dataset with the most packages) is keyed by bare folder, like the 2026-10-13
+    export; any other dataset as `<owner>/<name>/<folder>`."""
+    own = {o.lower() for o in own}
+    places = tree.get("qcodes") or {}
+    listings = listings or {}
+    packages: Dict[Tuple[str, str], Dict] = {}
+    for a in assets:
+        addr = package_address(a.get("url") or "")
+        if not addr:
+            continue
+        repo, rev, folder = addr
+        p = packages.setdefault((repo, folder), {"revisions": set(), "values": set(), "rules": set(), "files": 0})
+        p["files"] += 1
+        p["revisions"].add(rev)
+        p["values"].update(a.get("values") or [])
+        p["rules"].add(a.get("rule") or "none")
+    for (repo, rev), folders in listings.items():
+        if repo.split("/")[0].lower() not in own:
+            continue  # a partner's dataset holds more than its submission list: only listed packages count
+        for folder in folders:
+            packages.setdefault((repo, folder), {"revisions": set(), "values": set(), "rules": set(), "files": 0})
+            if not packages[(repo, folder)]["files"]:
+                packages[(repo, folder)]["revisions"].add(rev)
+
+    per_repo = Counter(repo for repo, _ in packages)
+    ours = [r for r in per_repo if r.split("/")[0].lower() in own]
+    primary = max(ours or per_repo, key=lambda r: (per_repo[r], r)) if per_repo else ""
+
+    rows: Dict[str, Dict] = {}
+    for (repo, folder), p in sorted(packages.items()):
+        values = sorted(p["values"])
+        revs = sorted(r for r in p["revisions"] if r)
+        commit = next((listings.get((repo, r), {}).get(folder) for r in revs if listings.get((repo, r), {}).get(folder)), None)
+        row = {"asset_path": folder, "repository": repo, "revision": revs[0] if len(revs) == 1 else None,
+               "last_commit": commit, "qcodes": [v for v in values if QCODE_RE.match(v)], "category_keys": values,
+               "rules": sorted(p["rules"]), "files": p["files"],
+               "category_path": None, "install_path": None, "status": "needs_review", "reason": None}
+        unplaced = [v for v in values if v not in places]
+        paths = {tree_folder(places[v]) for v in values if v in places}
+        if not p["files"]:
+            row["reason"] = "not_indexed"
+        elif len(revs) > 1:
+            row["reason"], row["revisions"] = "several_revisions", revs
+        elif not values:
+            row["reason"] = "missing_qcode"
+        elif unplaced:
+            row["reason"], row["unplaced"] = "qcode_not_in_tree", unplaced
+        elif None in paths:
+            row["reason"] = "no_category_folder"
+        elif len(paths) > 1:
+            row["reason"], row["category_paths"] = "multiple_category_paths", sorted(paths)
+        elif not safe_path(next(iter(paths))):
+            row["reason"], row["category_paths"] = "unsafe_category_path", sorted(paths)
+        else:
+            path = next(iter(paths))
+            row.update(category_path=path, install_path=f"{path}/{folder}", status="mapped")
+        rows[folder if repo == primary else f"{repo}/{folder}"] = row
+
+    by_dest: Dict[str, List[str]] = {}
+    for key, row in rows.items():
+        if row["install_path"]:
+            by_dest.setdefault(row["install_path"].casefold(), []).append(key)
+    for keys in by_dest.values():
+        if len(keys) > 1:  # two packages would land in one folder (a case-insensitive file system included)
+            for key in keys:
+                rows[key].update(category_paths=[rows[key]["category_path"]], category_path=None, install_path=None,
+                                 status="needs_review", reason="install_path_collision")
+
+    mapped = [r for r in rows.values() if r["status"] == "mapped"]
+    repos = sorted(per_repo)
+    revisions = {repo: dict(sorted(Counter(r["revision"] for r in rows.values()
+                                           if r["repository"] == repo and r["revision"]).items())) for repo in repos}
+    coverage = {
+        "assets": len(rows), "mapped_assets": len(mapped), "needs_review_assets": len(rows) - len(mapped),
+        "reasons": dict(sorted(Counter(r["reason"] for r in rows.values() if r["reason"]).items())),
+        "repositories": {repo: {"assets": sum(1 for r in rows.values() if r["repository"] == repo),
+                                "mapped_assets": sum(1 for r in mapped if r["repository"] == repo)} for repo in repos},
+        "assets_with_qcode": sum(1 for r in rows.values() if r["qcodes"]),
+        "qcodes": len({q for r in rows.values() for q in r["qcodes"]}),
+        "mapped_qcodes": len({q for r in mapped for q in r["qcodes"]}),
+        "unplaced_qcodes": sorted({v for r in rows.values() for v in r.get("unplaced") or []}),
+    }
+    primary_revs = list(revisions.get(primary) or {})
+    provenance = {
+        "repository": primary, "repository_url": f"https://huggingface.co/datasets/{primary}" if primary else None,
+        "revision": primary_revs[0] if len(primary_revs) == 1 else None, "revisions": revisions,
+        "corpus": corpus, "run_id": run_id,
+        "tree": {"file": HIERARCHY_FILE, "run_id": run_id, "generated_at": tree.get("generated_at"),
+                 "qcode_property": tree.get("qcode_property"), "source_qcodes": tree.get("source_qcodes"),
+                 "entries": len(places)},
+        "last_commits": last_commits or ("listed" if listings else "off"),
+        "method": f"each package's {PROPERTY} values (pipeline/categories.py) joined to the run's Category tree; "
+                  "a package maps only when every value sits under one folder path; nothing is guessed",
+    }
+    return {"schema_version": INSTALL_SCHEMA,
+            "generated_at": generated_at or dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat(),
+            "provenance": provenance, "coverage": coverage, "assets": rows}
+
+
+def write_install_list(out_dir: Path, tree: Dict, *, run_id: str = "", corpus: str = "") -> Optional[Path]:
+    """Write `install_categories.json` into the run's hierarchy dir from the categories report. None (and a
+    warning) when the report predates per-asset rows."""
+    report = json.loads(REPORT.read_text())
+    path = out_dir / INSTALL_FILE
+    if "assets" not in report:
+        path.unlink(missing_ok=True)  # never ship an earlier run's list with this run's tree
+        print(f"WARNING: {REPORT} has no per-asset rows (re-run the `categories` stage): no {INSTALL_FILE}")
+        return None
+    repos = {addr[:2] for addr in (package_address(a.get("url") or "") for a in report["assets"]) if addr}
+    if os.environ.get(LAST_COMMITS_ENV, "").strip().lower() == "off":
+        listings, note = {}, "off"
+    else:
+        listings, note = package_listings(repos)
+    doc = install_list(report["assets"], tree, own=load_vendors()["own"], listings=listings, run_id=run_id,
+                       corpus=corpus, last_commits=note)
+    path.write_text(json.dumps(doc, indent=1, ensure_ascii=False, sort_keys=True) + "\n")
+    cov = doc["coverage"]
+    print(f"{INSTALL_FILE}: {cov['mapped_assets']}/{cov['assets']} packages mapped; needs review {cov['reasons']}; "
+          f"last commits: {note}")
+    return path
+
+
 # ── stage: write the property into the live index ────────────────────────────
 
 
@@ -235,6 +450,7 @@ def run(cfg: Config, dry_run: bool = False) -> int:
     unplaced: List[str] = []
     updates: List[Tuple[str, List[Dict]]] = []
     urls: List[str] = []
+    rows: List[Dict] = []  # per file: the install list's input (install_list)
     sample_printed = False
     with httpx.Client(timeout=900.0) as http:
         for hit in _scroll(http, base, index):
@@ -252,6 +468,7 @@ def run(cfg: Config, dry_run: bool = False) -> int:
             vals, why = category_values(url, props or [], vendors, pins)
             reasons[why] += 1
             values.update(vals)
+            rows.append({"url": url, "values": vals, "rule": why})
             if not vals:
                 unplaced.append(url)
             new_props = with_category(props or [], vals)
@@ -275,7 +492,7 @@ def run(cfg: Config, dry_run: bool = False) -> int:
         REPORT.write_text(json.dumps({
             "property": PROPERTY, "index": index, "assets": len(urls), "rules": dict(reasons),
             "values": dict(sorted(values.items())), "unplaced": unplaced, "problems": problems,
-            "dry_run": dry_run,
+            "dry_run": dry_run, "assets": rows,
         }, indent=1) + "\n")
         if problems:
             print("ERROR: " + "\nERROR: ".join(problems))
@@ -342,6 +559,7 @@ def publish_hierarchy(cfg: Config, from_run: str, run_id: str, dry_run: bool = F
     doc = json.loads(tree_path.read_text())
     out, unplaced = augment_hierarchy(doc, report["values"].keys(), load_vendors(), load_pins())
     tree_path.write_text(json.dumps(out, indent=1, ensure_ascii=False, sort_keys=True) + "\n")
+    write_install_list(work, out, run_id=run_id, corpus=corpus)
     top = sorted({q["path"][0]["label"] for q in out["qcodes"].values() if q.get("path")})
     print(f"tree from {from_run}: {len(doc.get('qcodes') or {})} -> {len(out['qcodes'])} entries; top level {top}")
     if unplaced:
@@ -351,7 +569,7 @@ def publish_hierarchy(cfg: Config, from_run: str, run_id: str, dry_run: bool = F
         return 0
     s3.upload_dir(cfg, work, dest)
     landed = {rel for rel, _ in s3.list_objects(cfg, dest)}
-    expected = {rel for rel, _ in objects}
+    expected = {p.relative_to(work).as_posix() for p in work.rglob("*") if p.is_file()}
     if landed != expected:
         print(f"ERROR: {dest} holds {sorted(landed)}, expected {sorted(expected)}")
         return 1

pipeline/config.py

@@ -107,6 +107,7 @@ class Config:
             "INFO_ENDPOINT_URL",
             "OPENSEARCH_URL",
             "SIGLIP2_BACKEND",
+            "INDEXING_SIGLIP2_BACKEND",
             "SIGLIP2_NVCF_ENDPOINT",
             "STORAGE_BACKEND_TYPE",
             "SIGLIP2_NVCF_FUNCTION_ID",
@@ -211,6 +212,24 @@ class Config:
     def siglip2_nvcf(self) -> bool:
         return self.siglip2_backend == "nvcf"
 
+    # SIGLIP2_BACKEND also decides what serves queries (the verify stack, the
+    # dev Space's NVCF-only hf_space image), so the indexing stack may override
+    # it: e.g. embed the corpus on the local GPU while the dev Space keeps
+    # querying NVCF. Same model either way, so the vectors are interchangeable.
+
+    @property
+    def indexing_siglip2_backend(self) -> str:
+        backend = self.get("INDEXING_SIGLIP2_BACKEND").strip().lower()
+        if not backend:
+            return self.siglip2_backend
+        if backend not in ("local", "nvcf"):
+            raise SystemExit(f"INDEXING_SIGLIP2_BACKEND must be 'local' or 'nvcf', got {backend!r}")
+        return backend
+
+    @property
+    def indexing_siglip2_nvcf(self) -> bool:
+        return self.indexing_siglip2_backend == "nvcf"
+
     # ── HF Space GPU variant (local siglip2-triton, no NVCF) ────────────────
     # Independent of SIGLIP2_BACKEND above (which is the *indexing stack's*
     # compose backend, local-GPU vs. NVCF) — this is a second, optional
@@ -451,6 +470,23 @@ class Config:
                 "../usdsearch/docs/models-and-config.md)."
             )
 
+    @staticmethod
+    def hf_token() -> str:
+        """$HF_TOKEN, else the token `hf auth login` cached — the same lookup
+        the enumerator's huggingface_hub does on its own. The container only
+        sees what compose_env() passes it, so without this fallback it ran
+        anonymously whenever $HF_TOKEN wasn't exported, and a public dataset
+        still "worked": HF's anonymous per-IP API quota (per 5-minute window)
+        throttled a 2,600-package re-index to ~2 asset graphs/min with 429s."""
+        token = os.environ.get("HF_TOKEN", "").strip()
+        if token:
+            return token
+        try:
+            from huggingface_hub import get_token
+        except ImportError:
+            return ""
+        return (get_token() or "").strip()
+
     def compose_env(self) -> Dict[str, str]:
         """The environment every `docker compose`/`docker run` subprocess
         this pipeline spawns should get — `pipeline.env`'s values plus
@@ -482,7 +518,7 @@ class Config:
             # so operators only ever set one variable. Verified live: without
             # this, hf-storage-api silently ran unauthenticated and every
             # submission against a private dataset 401'd as "Asset missing".
-            hf_token = os.environ.get("HF_TOKEN", "").strip()
+            hf_token = self.hf_token()
             if hf_token:
                 env["HF_ADAPTER_HF_TOKEN"] = hf_token
         # No vlm_metadata_enabled-specific merge needed here: `env` already
@@ -496,7 +532,7 @@ class Config:
 
     def indexing_compose_files(self) -> List[str]:
         files = ["-f", str(REPO_ROOT / "compose" / "docker-compose.indexing.yml")]
-        if self.siglip2_nvcf:
+        if self.indexing_siglip2_nvcf:
             files += ["-f", str(REPO_ROOT / "compose" / "docker-compose.indexing.nvcf-siglip2.yml")]
         # Only the indexing stack ever talks to a storage backend directly —
         # the runtime stack (verify/HF Space) serves an already-baked index and

pipeline/corpus.py

@@ -9,6 +9,8 @@ exactly the kind of thing that silently indexes the wrong 200k assets.
 
 from __future__ import annotations
 
+import bisect
+import functools
 import hashlib
 import json
 from dataclasses import dataclass
@@ -46,6 +48,25 @@ class CorpusError(ValueError):
     """Raised for a corpus file that cannot be used as written."""
 
 
+def _prefix_index(uris: Iterable[str]) -> List[str]:
+    """Sorted, with every entry another entry already covers dropped — so at
+    most one entry is a prefix of any string (see `_has_prefix_in`)."""
+    index: List[str] = []
+    for uri in sorted(set(uris)):
+        if not (index and uri.startswith(index[-1])):
+            index.append(uri)
+    return index
+
+
+def _has_prefix_in(index: List[str], uri: str) -> bool:
+    """Whether some entry of `index` (from `_prefix_index`) is a prefix of
+    `uri`. Only the greatest entry <= `uri` can be: any entry between a prefix
+    p and `uri` either starts with p (dropped from the index) or sorts after
+    `uri`."""
+    i = bisect.bisect_right(index, uri)
+    return i > 0 and uri.startswith(index[i - 1])
+
+
 @dataclass(frozen=True)
 class S3Prefix:
     bucket: str
@@ -201,13 +222,25 @@ class Corpus:
 
     # ── matching ──────────────────────────────────────────────────────────
 
+    # `enumerate` tests every file of every listed repo tree (400k on 2026-10-19)
+    # against every include (3,569), so a linear startswith scan took ~25 min.
+    # Same literal-prefix semantics, one bisect per lookup: see _prefix_index.
+
+    @functools.cached_property
+    def _include_index(self) -> List[str]:
+        return _prefix_index(p.uri for p in self.include)
+
+    @functools.cached_property
+    def _exclude_index(self) -> List[str]:
+        return _prefix_index(p.uri for p in self.exclude)
+
     def is_excluded(self, uri: str) -> bool:
         if any(s in uri for s in ALWAYS_EXCLUDE_SUBSTRINGS):
             return True
-        return any(uri.startswith(p.uri) for p in self.exclude)
+        return _has_prefix_in(self._exclude_index, uri)
 
     def is_included(self, uri: str) -> bool:
-        return any(uri.startswith(p.uri) for p in self.include)
+        return _has_prefix_in(self._include_index, uri)
 
     def has_wanted_extension(self, uri: str) -> bool:
         lowered = uri.lower()
@@ -318,13 +351,15 @@ class Corpus:
 
         # A prefix nested inside another include is redundant, not wrong — but
         # it usually signals a misunderstanding of the trailing-slash rule.
-        for a in self.include:
-            for b in self.include:
-                if a is not b and a.uri != b.uri and a.uri.startswith(b.uri):
-                    raise CorpusError(
-                        f"include: {a.uri!r} is already covered by {b.uri!r}; "
-                        f"remove the redundant entry"
-                    )
+        # Sorted, a prefix's immediate successor starts with it whenever any
+        # entry does, so adjacent pairs are enough (duplicates are gone above).
+        uris = sorted(seen)
+        for b, a in zip(uris, uris[1:]):
+            if a.startswith(b):
+                raise CorpusError(
+                    f"include: {a!r} is already covered by {b!r}; "
+                    f"remove the redundant entry"
+                )
 
         # A stray exclude — one that sits under no include — is nearly always a
         # typo, and it silently does nothing.

pipeline/enumerate_assets.py

@@ -131,6 +131,33 @@ def _hf_api():
     return HfApi(token=os.environ.get("HF_TOKEN") or None)
 
 
+PACKAGE_SIZES = BUILD_DIR / "package_sizes.json"
+
+
+def _package_sizes_group(repo_key: hf_catalog.RepoKey, file_entries: Dict) -> Dict:
+    """One (repo, revision)'s sums for the Space's /package_sizes.json, from the listing enumerate already made:
+    the whole repo's bytes and each top-level folder's — every file, not only the matched ones (same definition
+    as pipeline/package_download/sizes.py's own listing, which this spares at Space start)."""
+    _host, repo_type, repo_id, revision = repo_key
+    whole, folders = 0, {}
+    for path, entry in file_entries.items():
+        size = int(entry.size or 0)
+        whole += size
+        if "/" in path:
+            top = path.split("/", 1)[0]
+            folders[top] = folders.get(top, 0) + size
+    return {"kind": f"{repo_type}s", "repo_id": repo_id, "revision": revision, "whole": whole,
+            "folders": folders, "files": len(file_entries)}
+
+
+def write_package_sizes(groups: List[Dict]) -> None:
+    """`export` ships this with the run (next to hf-storage-api-catalog.yaml); the Space's sizes task loads it
+    from the restore and lists HF only for a revision it lacks."""
+    doc = {"version": 1, "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "groups": groups}
+    PACKAGE_SIZES.write_text(json.dumps(doc, separators=(",", ":"), sort_keys=True))
+    print(f"  wrote {PACKAGE_SIZES.name}: {len(groups)} (repo, revision) group(s)")
+
+
 def enumerate_corpus_hf(corpus: Corpus, cfg: Config) -> Tuple[List[Asset], Dict[hf_catalog.RepoKey, List[str]]]:
     """The `storage_api` backend's counterpart to `enumerate_corpus` — lists
     each corpus HF repo's full file tree, applies the same include/exclude/
@@ -144,6 +171,7 @@ def enumerate_corpus_hf(corpus: Corpus, cfg: Config) -> Tuple[List[Asset], Dict[
     seen: Dict[str, Asset] = {}
     matched_paths_by_repo: Dict[hf_catalog.RepoKey, List[str]] = {}
     all_paths_by_repo: Dict[hf_catalog.RepoKey, Set[str]] = {}
+    package_sizes: List[Dict] = []
     listed = 0
 
     for repo_key in corpus.hf_repos:
@@ -155,6 +183,7 @@ def enumerate_corpus_hf(corpus: Corpus, cfg: Config) -> Tuple[List[Asset], Dict[
             if isinstance(entry, RepoFile)
         }
         all_paths_by_repo[repo_key] = set(file_entries)
+        package_sizes.append(_package_sizes_group(repo_key, file_entries))
 
         matched: List[str] = []
         for path, entry in file_entries.items():
@@ -176,6 +205,7 @@ def enumerate_corpus_hf(corpus: Corpus, cfg: Config) -> Tuple[List[Asset], Dict[
             print(f"  dropped {len(dropped)} 'removed' asset(s) from submission")
 
     print(f"  listed {listed} objects, accepted {len(seen)}")
+    write_package_sizes(package_sizes)
     asset_folders = hf_catalog.asset_folders_by_repo(corpus, matched_paths_by_repo, all_paths_by_repo)
     return [seen[k] for k in sorted(seen)], asset_folders
 
@@ -246,6 +276,7 @@ def run(cfg: Config, dry_run: bool = False) -> int:
     BUILD_DIR.mkdir(parents=True, exist_ok=True)
 
     asset_folders: Dict[hf_catalog.RepoKey, List[str]] = {}
+    PACKAGE_SIZES.unlink(missing_ok=True)  # an earlier run's sums never ride along with this one
     if corpus.storage_backend == "storage_api":
         print("Enumerating HuggingFace ...")
         assets, asset_folders = enumerate_corpus_hf(corpus, cfg)

pipeline/export.py

@@ -218,9 +218,14 @@ def export_neo4j(cfg: Config) -> Dict:
     print("  stopping asset-graph-service ...")
     _run(["docker", "compose", "-p", COMPOSE_PROJECT, *cfg.indexing_compose_files(),
           "stop", "--timeout", "300", "asset-graph-service"], env=cfg.compose_env())
-    print("  stopping neo4j ...")
+    # 300 s wasn't enough either on 2026-10-16: the full simready-assets graph
+    # (~149k nodes, ~12k property indexes) was SIGKILLed mid-checkpoint and the
+    # dump refused with "Active logical log detected". Recover by starting
+    # neo4j once (it replays the log) and rerunning export.
+    stop_timeout = cfg.get_int("NEO4J_STOP_TIMEOUT", 1800)
+    print(f"  stopping neo4j (up to {stop_timeout}s for its shutdown checkpoint) ...")
     _run(["docker", "compose", "-p", COMPOSE_PROJECT, *cfg.indexing_compose_files(),
-          "stop", "--timeout", "300", "neo4j"], env=cfg.compose_env())
+          "stop", "--timeout", str(stop_timeout), "neo4j"], env=cfg.compose_env())
 
     print("  dumping neo4j ...")
     # Runs as root (the image default), NOT as the host uid: the data volume is
@@ -276,6 +281,18 @@ def run(cfg: Config) -> int:
 
     print("Exporting OpenSearch ...")
     manifest["opensearch_artifacts"] = export_opensearch(cfg, manifest)
+    from .image_overrides import check_dump  # every doc its own nonzero vector, >= 98 % distinct
+    main_dump = OS_ARTIFACTS / f"{cfg.main_index}.data.json.gz"
+    if main_dump.exists() and check_dump(main_dump) != 0:
+        print(f"export: {main_dump.name} fails the vector check — not exporting a broken index")
+        return 1
+
+    # `wait` recorded image_cache_docs before `image-overrides` ran, and each override adds an image document
+    # (the package's own thumbnail document stays): 2597 assets -> 2605 image docs on 2026-10-16. scripts/smoke.sh
+    # checks the baked image against this number, so take it from what was actually exported.
+    image_cache = manifest["opensearch_artifacts"].get(cfg.image_cache_index)
+    if image_cache is not None:
+        manifest["image_cache_docs"] = image_cache["doc_count"]
 
     print("\nExporting Neo4j ...")
     manifest["neo4j_artifact"] = export_neo4j(cfg)
@@ -289,10 +306,14 @@ def run(cfg: Config) -> int:
     # docs/hf-space-deployment.md §4 item 3 for why this is a
     # frozen index-time artifact, not something restored by re-listing the
     # HF dataset at Space startup.
-    for name in ("corpus.json", "index-report.json", "assets.jsonl", "hf-storage-api-catalog.yaml"):
+    # package_sizes.json (storage_api too): the Space's /package_sizes.json sums, from enumerate's own listing.
+    for name in ("corpus.json", "index-report.json", "assets.jsonl", "hf-storage-api-catalog.yaml",
+                 "package_sizes.json"):
         src = BUILD_DIR / name
         if src.exists():
             shutil.copy2(src, ARTIFACTS / name)
+        else:
+            (ARTIFACTS / name).unlink(missing_ok=True)  # .build/artifacts/ outlives runs; never ship an older one's
 
     save_manifest(manifest)
     shutil.copy2(BUILD_DIR / "manifest.json", ARTIFACTS / "manifest.json")

pipeline/hf_space.py

@@ -51,7 +51,7 @@ from typing import Dict, List
 from huggingface_hub import HfApi, SpaceHardware
 from huggingface_hub.errors import HfHubHTTPError
 
-from .artifact_store import space_artifact_secrets
+from .artifact_store import bucket_base, list_objects, space_artifact_secrets
 from .bake import DOCKER_DIR, hf_space_build_args, hf_space_gpu_build_args, image_tags
 from .config import BUILD_DIR, HF_SPACE_GPU_HARDWARE_DEFAULT, REPO_ROOT, Config
 from .corpus import Corpus
@@ -225,6 +225,22 @@ def stage_repo(dest: Path, marker: str, readme: str, dockerfile_name: str = "Doc
     shutil.copytree(REPO_ROOT / "services" / "explorer" / "dist", dest / "services" / "explorer" / "dist")
 
 
+def hierarchy_missing(cfg: Config, manifest: Dict) -> bool:
+    """True when the run has no published category tree. The Space serves the run's
+    hierarchy/category_hierarchy.json as /wikidata_hierarchy.json (the Explorer's Category tree), so a run without
+    one deploys a Space with no category browsing. `all --stop verify` skips the `hierarchy` stage; run it with
+    `--run-id <run>` before deploying."""
+    prefix = f"{bucket_base(cfg)}/{manifest['corpus_name']}/{manifest['run_id']}/hierarchy"
+    if any(rel == "category_hierarchy.json" for rel, _ in list_objects(cfg, prefix)):
+        return False
+    print(
+        f"Run {manifest['run_id']} has no {prefix}/category_hierarchy.json — the `hierarchy` stage is required "
+        f"before deploying. Run: python -m pipeline.run_pipeline hierarchy --env-file ... "
+        f"--run-id {manifest['run_id']} --new-only --api-url <a running stack>"
+    )
+    return True
+
+
 def run(cfg: Config) -> int:
     manifest = load_manifest()
     if not manifest.get("hf_space_smoke_passed"):
@@ -238,6 +254,8 @@ def run(cfg: Config) -> int:
     if "hf_space" not in image_tags(cfg, manifest):
         print("No hf_space image for this run (SIGLIP2_BACKEND != nvcf) — nothing to deploy.")
         return 1
+    if hierarchy_missing(cfg, manifest):
+        return 1
 
     repo_id = cfg.require("HF_SPACE_REPO_ID")  # e.g. "nvidia/simready-central"
     title = cfg.get("HF_SPACE_TITLE", repo_id.split("/", 1)[-1])
@@ -329,6 +347,8 @@ def run_gpu(cfg: Config) -> int:
     if "hf_space_gpu" not in image_tags(cfg, manifest):
         print("No hf_space_gpu image for this run (HF_SPACE_BUILD_GPU_VARIANT != true) — nothing to deploy.")
         return 1
+    if hierarchy_missing(cfg, manifest):
+        return 1
 
     repo_id = cfg.require("HF_SPACE_REPO_ID_GPU")  # e.g. "nvidia/simready-central-gpu"
     title = cfg.get("HF_SPACE_TITLE_GPU", repo_id.split("/", 1)[-1])

pipeline/hierarchy/run.py

@@ -250,8 +250,11 @@ def run(cfg: Config, *, api_url: Optional[str] = None, replace_all: bool = False
                                                      categories.load_vendors(), categories.load_pins())
         if unplaced:
             print(f"WARNING: {len(unplaced)} {categories.PROPERTY} value(s) without a place in the tree: {unplaced}")
+        # Package folder -> category folder for installers, from the same tree (served as /install_categories.json).
+        categories.write_install_list(out_dir, doc, run_id=target.rsplit("/", 1)[-1] if target else "", corpus=corpus)
     else:
         print(f"WARNING: no {categories.REPORT} (the `categories` stage did not run): the tree keeps {qprop}")
+        (out_dir / categories.INSTALL_FILE).unlink(missing_ok=True)
     (out_dir / "category_hierarchy.json").write_text(json.dumps(doc, indent=1, ensure_ascii=False, sort_keys=True) + "\n")
     cache_doc = {"corpus": corpus, "generated_at": now, "qcode_property": qprop, "metrics": metrics, "placements": {
         q: {"path": p.path, "label": p.label, "confidence": round(p.confidence, 3), "reason": p.reason,

pipeline/hierarchy/store.py

@@ -2,6 +2,7 @@
 
     <ARTIFACT_S3_URI>/<corpus>/<run_id>/hierarchy/
         category_hierarchy.json   the Explorer's tree (the Space restores it with the run)
+        install_categories.json   package folder -> category folder (pipeline/categories.py install_list)
         placements.json           decision cache: reused by the next run
         folders_added.txt         folders the agent created so far (base = repo file + these)
         report.md, report.html, misfiled_assets.json   for review

pipeline/image_overrides.py

@@ -0,0 +1,190 @@
+"""Stage `image-overrides` — embed, thumbnail and caption chosen assets from OUR image instead of the package's.
+
+A partner package's own `.thumbs/256x256/` thumbnail can be a poor picture of it (a flat render on white, a cropped
+part); its SigLIP2 embedding then drifts into unrelated searches. config/image_overrides/overrides.yaml names, per
+asset (its hf:// address in the corpus), an image in that folder — SimReady Central's RTX still or spin start frame.
+After `wait`, this stage runs usdsearch's OWN plugins on that image, inside the usdsearch image (same SigLIP2 client,
+same VLM metadata generation and field schema, same OpenSearch writer as the workers), for each listed asset:
+
+  * thumbnail_to_embedding: the embedding and the stored thumbnail (what /images serves) are replaced;
+  * thumbnail_to_vision_metadata: the VLM metadata (caption, tags, ...) is written — only when IMAGE_OVERRIDES_VLM is
+    on, with USDSEARCH_LLM_BASE_URL / USDSEARCH_LLM_API_KEY / USDSEARCH_VISION_METADATA_MODEL from the environment.
+
+Every call has a timeout and retries (IMAGE_OVERRIDES_TIMEOUT_S, IMAGE_OVERRIDES_TRIES); any asset left without its
+embedding, or without its metadata when VLM is on, fails the stage. A wide image is padded to a square first (the
+SigLIP2 preprocessing squashes to 384x384). The record goes to .build/image-overrides.json.
+
+    python -m pipeline.run_pipeline image-overrides --env-file ... [--dry-run]
+
+An entry may also name `vlm_image`: the picture the VLM describes instead of `image` (e.g. an appliance shown open).
+REEMBED_FROM_THUMBNAILS=true additionally recomputes every OTHER asset's embedding from its own `.thumbs` thumbnail
+through the plugin's own loader (batched: REEMBED_BATCH, REEMBED_CONCURRENCY) and then checks that every document
+carries its own vector (none zero, at least 98 % distinct) — the repair for vectors a restore left unusable.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Dict, List
+
+import yaml
+
+from .config import BUILD_DIR, REPO_ROOT, Config
+
+OVERRIDES_DIR = REPO_ROOT / "config" / "image_overrides"
+SPEC_FILE = OVERRIDES_DIR / "overrides.yaml"
+REPORT = BUILD_DIR / "image-overrides.json"
+WORKER = Path(__file__).with_name("image_overrides_worker.py")
+COMPOSE_PROJECT = "simready-indexing"
+SERVICE = "omni-writer"  # has the OpenSearch writer's settings; the SigLIP2/LLM env is added per run
+IMAGE_EXTS = (".jpg", ".jpeg", ".png", ".webp")
+
+
+def load_spec(path: Path = SPEC_FILE, includes: List[str] | None = None) -> Dict[str, Dict]:
+    """{asset address: {image, source}} — every image present in the folder, every asset in the corpus (when given)."""
+    if not path.exists():
+        return {}
+    raw = yaml.safe_load(path.read_text()) or {}
+    out: Dict[str, Dict] = {}
+    problems: List[str] = []
+    include_set = set(includes) if includes is not None else None
+    for key, rec in raw.items():
+        rec = rec if isinstance(rec, dict) else {"image": rec}
+        image = str(rec.get("image") or "")
+        if not key.startswith("hf://"):
+            problems.append(f"{key}: not an hf:// asset address")
+        if not image or "/" in image or not image.lower().endswith(IMAGE_EXTS):
+            problems.append(f"{key}: image {image!r} must be a file name in {OVERRIDES_DIR.name}/ ({', '.join(IMAGE_EXTS)})")
+        elif not (path.parent / image).is_file():
+            problems.append(f"{key}: {image} is missing")
+        if include_set is not None and key not in include_set:
+            problems.append(f"{key}: not in the corpus")
+        vlm_image = str(rec.get("vlm_image") or "")
+        if vlm_image and ("/" in vlm_image or not vlm_image.lower().endswith(IMAGE_EXTS) or not (path.parent / vlm_image).is_file()):
+            problems.append(f"{key}: vlm_image {vlm_image!r} must be an image file in {OVERRIDES_DIR.name}/")
+        out[key] = {"image": image, "source": str(rec.get("source") or "")}
+        if vlm_image:
+            out[key]["vlm_image"] = vlm_image
+    if problems:
+        raise SystemExit("image overrides:\n  " + "\n  ".join(problems))
+    return out
+
+
+def vlm_enabled() -> bool:
+    return os.environ.get("IMAGE_OVERRIDES_VLM", "false").strip().lower() in ("1", "true", "yes", "on")
+
+
+def run(cfg: Config, dry_run: bool = False) -> int:
+    corpus_path = BUILD_DIR / "corpus.json"
+    includes = json.loads(corpus_path.read_text()).get("include") if corpus_path.exists() else None
+    spec = load_spec(includes=includes)
+    if not spec:
+        print(f"no image overrides ({SPEC_FILE.relative_to(REPO_ROOT)} empty or absent) — nothing to do")
+        return 0
+    vlm = vlm_enabled()
+    if vlm and not os.environ.get("USDSEARCH_LLM_API_KEY", "").strip():
+        print("IMAGE_OVERRIDES_VLM is on but USDSEARCH_LLM_API_KEY is not set in the environment")
+        return 1
+    print(f"{len(spec)} image override(s); VLM metadata {'on (' + os.environ.get('USDSEARCH_VISION_METADATA_MODEL', 'default model') + ')' if vlm else 'off'}")
+    for key, rec in spec.items():
+        print(f"  {rec['image']:<44} {key}")
+    if dry_run:
+        print("dry run: nothing written")
+        return 0
+
+    BUILD_DIR.mkdir(parents=True, exist_ok=True)
+    spec_path = BUILD_DIR / "image-overrides-spec.json"
+    reembed = []
+    if os.environ.get("REEMBED_FROM_THUMBNAILS", "").strip().lower() in ("1", "true", "yes", "on"):
+        # Repair: every other asset's embedding is recomputed from its own .thumbs thumbnail (the plugin's own loader).
+        reembed = sorted(k for k in (includes or []) if k not in spec)
+        print(f"re-embedding {len(reembed)} other asset(s) from their own thumbnails")
+    spec_path.write_text(json.dumps({"index": cfg.main_index, "vlm": vlm, "assets": spec, "reembed": reembed}, indent=1) + "\n")
+    passthrough = [
+        "SIGLIP2_TRITON_SERVER_URL", "SIGLIP2_TRITON_SERVER_SSL", "SIGLIP2_TRITON_SERVER_AUTH_TOKEN",
+        "SIGLIP2_TRITON_SERVER_HEADERS", "USDSEARCH_LLM_BASE_URL", "USDSEARCH_LLM_API_KEY",
+        "USDSEARCH_VISION_METADATA_MODEL", "IMAGE_OVERRIDES_TIMEOUT_S", "IMAGE_OVERRIDES_TRIES",
+        "REEMBED_BATCH", "REEMBED_CONCURRENCY",
+    ]
+    env = cfg.compose_env()
+    cmd = ["docker", "compose", "-p", COMPOSE_PROJECT, *cfg.indexing_compose_files(), "run", "--rm", "--no-deps", "-T",
+           "-v", f"{WORKER}:/ops/image_overrides_worker.py:ro",
+           "-v", f"{OVERRIDES_DIR}:/overrides:ro",
+           "-v", f"{spec_path}:/ops/spec.json:ro",
+           "-v", f"{REPO_ROOT / 'config' / 'vlm_metadata_fields.yaml'}:/app/vlm_metadata_fields.yaml:ro",
+           "-e", "USDSEARCH_VISION_METADATA_YAML_FILEPATH=/app/vlm_metadata_fields.yaml",
+           "-e", f"OPENSEARCH_URL_INTERNAL=http://opensearch:9200"]
+    for var in passthrough:
+        if env.get(var):
+            cmd += ["-e", var]  # value from the compose CLI's environment, never on the command line
+    cmd += [SERVICE, "python", "/ops/image_overrides_worker.py", "/ops/spec.json"]
+    print("$ docker compose ... run --rm " + SERVICE + " python /ops/image_overrides_worker.py")
+    proc = subprocess.run(cmd, env=env, capture_output=True, text=True)
+    sys.stdout.write(proc.stderr[-20000:] if proc.returncode else "")
+    report = None
+    for line in proc.stdout.splitlines():
+        if line.startswith("IMAGE_OVERRIDES_REPORT "):
+            report = json.loads(line[len("IMAGE_OVERRIDES_REPORT "):])
+        else:
+            print(line)
+    if report is None:
+        print(f"the worker wrote no report (exit {proc.returncode})")
+        return 1
+    REPORT.write_text(json.dumps(report, indent=1) + "\n")
+    failed = [k for k, r in report["assets"].items() if not r.get("ok")]
+    print(f"image overrides: {len(report['assets']) - len(failed)} ok, {len(failed)} failed")
+    if failed or proc.returncode:
+        for k in failed:
+            print(f"  FAILED {k}: {report['assets'][k].get('error')}")
+        return 1
+    return 0
+
+
+def check_dump(path: Path, min_distinct: float = 0.98) -> int:
+    """Gate on an exported OpenSearch dump (elasticdump layout, .data.json[.gz]): every document carries a nonzero
+    siglip2 vector and at least `min_distinct` of them are distinct. Run 20261012-9d9f42a's dump failed this: one
+    vector on 1,399 of 1,404 documents."""
+    import gzip
+    import hashlib
+
+    def first_vector(o):
+        if isinstance(o, list) and o and all(isinstance(x, (int, float)) for x in o[:4]) and len(o) >= 64:
+            return o
+        items = o.values() if isinstance(o, dict) else (o if isinstance(o, list) else [])
+        for v in items:
+            found = first_vector(v)
+            if found is not None:
+                return found
+        return None
+
+    opener = gzip.open if str(path).endswith(".gz") else open
+    docs = missing = zero = 0
+    counts: Dict[str, int] = {}
+    with opener(path, "rt") as fh:
+        for line in fh:
+            if not line.strip():
+                continue
+            src = json.loads(line).get("_source") or {}
+            docs += 1
+            v = first_vector(src.get("siglip2-embedding"))
+            if v is None:
+                missing += 1
+                continue
+            if sum(x * x for x in v) < 1e-12:
+                zero += 1
+            h = hashlib.sha1(json.dumps([round(x, 5) for x in v]).encode()).hexdigest()
+            counts[h] = counts.get(h, 0) + 1
+    distinct = len(counts)
+    largest = max(counts.values(), default=0)
+    print(f"{path.name}: {docs} docs, {missing} without a vector, {zero} zero, {distinct} distinct, largest shared group {largest}")
+    return 0 if docs and not missing and not zero and distinct >= min_distinct * docs else 1
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 3 and sys.argv[1] == "check-dump":
+        sys.exit(check_dump(Path(sys.argv[2])))
+    sys.exit("usage: python -m pipeline.image_overrides check-dump <...data.json[.gz]>")

pipeline/image_overrides_worker.py

@@ -0,0 +1,291 @@
+"""Runs INSIDE the usdsearch image (pipeline/image_overrides.py starts it with `docker compose run`): usdsearch's own
+thumbnail_to_embedding and thumbnail_to_vision_metadata plugins on our image for each listed asset, written with the
+plugins' own add_metadata through the same storage client the omni-writer uses. Prints one line
+`IMAGE_OVERRIDES_REPORT {json}` at the end. Stdlib + what the usdsearch image ships (PIL, plugins, storage).
+"""
+
+import asyncio
+import hashlib
+import json
+import os
+import sys
+import time
+import traceback
+import urllib.request
+
+from PIL import Image
+
+TIMEOUT_S = float(os.environ.get("IMAGE_OVERRIDES_TIMEOUT_S") or 180)
+TRIES = int(os.environ.get("IMAGE_OVERRIDES_TRIES") or 3)
+OS_URL = os.environ.get("OPENSEARCH_URL_INTERNAL", "http://opensearch:9200").rstrip("/")
+THUMB_PX = 256          # the size every other asset's thumbnail is embedded and stored at (.thumbs/256x256)
+VLM_PX = 512            # the VLM plugin's own vlm_image_max_px default
+
+
+def square(img: Image.Image) -> Image.Image:
+    """Pad a wide or tall image to a square with its own border colour (the SigLIP2 preprocessing squashes to a
+    square, which would distort it)."""
+    img = img.convert("RGB")
+    w, h = img.size
+    if w == h:
+        return img
+    side = max(w, h)
+    px = img.load()
+    border = [px[x, 0] for x in range(w)] + [px[x, h - 1] for x in range(w)] + [px[0, y] for y in range(h)] + [px[w - 1, y] for y in range(h)]
+    fill = tuple(sum(c[i] for c in border) // len(border) for i in range(3))
+    out = Image.new("RGB", (side, side), fill)
+    out.paste(img, ((side - w) // 2, (side - h) // 2))
+    return out
+
+
+def os_doc(index: str, key: str) -> dict:
+    body = json.dumps({"size": 2, "query": {"term": {"base_key": key}}}).encode()
+    req = urllib.request.Request(f"{OS_URL}/{index}/_search", data=body, headers={"content-type": "application/json"})
+    hits = json.load(urllib.request.urlopen(req, timeout=60))["hits"]["hits"]
+    if len(hits) != 1:
+        raise RuntimeError(f"{len(hits)} documents for {key}")
+    return hits[0]["_source"]
+
+
+def digests(src: dict) -> dict:
+    return {k: hashlib.sha256(json.dumps(v, sort_keys=True, default=str).encode()).hexdigest()[:12] for k, v in src.items()}
+
+
+def result_content(results: dict, plugin_name: str):
+    r = results.get(0)
+    if r is None:
+        raise RuntimeError(f"{plugin_name}: no result")
+    content = (r.get("search_backend_content") or {}).get(plugin_name)
+    if not content:
+        raise RuntimeError(f"{plugin_name}: empty content ({r.get('asset_status')})")
+    if "error" in content:
+        raise RuntimeError(f"{plugin_name}: {content['error']}")
+    return content
+
+
+async def with_retries(what: str, fn):
+    last = None
+    for attempt in range(1, TRIES + 1):
+        try:
+            return await asyncio.wait_for(fn(), timeout=TIMEOUT_S)
+        except Exception as exc:  # noqa: BLE001 — retried, then reported
+            last = f"{type(exc).__name__}: {exc}"
+            print(f"    {what}: attempt {attempt}/{TRIES} failed: {last}", flush=True)
+            await asyncio.sleep(5 * attempt)
+    raise RuntimeError(f"{what}: gave up after {TRIES} attempts: {last}")
+
+
+def thumbnail_url(key: str) -> str:
+    """The asset's own .thumbs/256x256 thumbnail at its pinned revision — the file the thumbnail_to_embedding worker
+    embeds: hf://<host>/datasets/<o>/<n>/resolve/<rev>/<dir>/<file> -> https://<host>/datasets/.../<dir>/.thumbs/256x256/<file>.png"""
+    if not key.startswith("hf://"):
+        raise ValueError(f"not an hf:// address: {key}")
+    folder, name = key[len("hf://"):].rsplit("/", 1)
+    return f"https://{folder}/.thumbs/256x256/{name}.png"
+
+
+def fetch_thumbnail(key: str) -> Image.Image:
+    import io
+    last = None
+    for attempt in range(1, 6):
+        try:
+            req = urllib.request.Request(thumbnail_url(key), headers={"User-Agent": "simready-central-indexing/reembed"})
+            with urllib.request.urlopen(req, timeout=60) as f:
+                img = Image.open(io.BytesIO(f.read()))
+                img.load()
+                return img.convert("RGB")
+        except urllib.error.HTTPError as exc:
+            last = exc
+            if exc.code not in (429, 500, 502, 503, 504):
+                raise
+        except Exception as exc:  # noqa: BLE001
+            last = exc
+        time.sleep(2 * attempt)
+    raise RuntimeError(f"thumbnail {thumbnail_url(key)}: {last}")
+
+
+def served_thumbnail(index: str, key: str) -> Image.Image:
+    """The thumbnail the Space serves for `key` (/images): the image-cache document the doc's siglip2-embedding
+    entry points at (its `image` id), decoded."""
+    import base64
+    import io
+    src = os_doc(index, key)
+    entries = src.get("siglip2-embedding") or []
+    image_id = entries[0].get("image") if entries and isinstance(entries[0], dict) else None
+    if not image_id:
+        raise RuntimeError("no image id on the document")
+    cache_index = index.replace("-ver5.0", "-ver4.0-image-cache")
+    doc = json.load(urllib.request.urlopen(f"{OS_URL}/{cache_index}/_doc/{image_id}", timeout=60))["_source"]
+    b64 = next(iter(doc.values()))
+    img = Image.open(io.BytesIO(base64.b64decode(b64)))
+    img.load()
+    return img.convert("RGB")
+
+
+def same_picture(a: Image.Image, b: Image.Image) -> float:
+    """Mean absolute difference (0-255) of two pictures at 64x64 greyscale: JPEG re-encoding stays well under 10."""
+    pa = list(a.convert("L").resize((64, 64), Image.BILINEAR).getdata())
+    pb = list(b.convert("L").resize((64, 64), Image.BILINEAR).getdata())
+    return sum(abs(x - y) for x, y in zip(pa, pb)) / len(pa)
+
+
+def check_sources(index: str, keys: list, report: dict, sample: int = 20, max_diff: float = 12.0) -> bool:
+    """Before re-embedding: on a sample, the fetched .thumbs file must be the picture the Space serves."""
+    step = max(1, len(keys) // sample)
+    rows, bad = [], 0
+    for key in keys[::step][:sample]:
+        try:
+            fetched, served = fetch_thumbnail(key), served_thumbnail(index, key)
+            d = round(same_picture(fetched, served), 2)
+            rows.append({"key": key, "fetched": list(fetched.size), "served": list(served.size), "diff": d})
+            bad += d > max_diff
+        except Exception as exc:  # noqa: BLE001
+            rows.append({"key": key, "error": f"{type(exc).__name__}: {exc}"})
+            bad += 1
+    report["source_check"] = {"sample": len(rows), "mismatched": bad, "max_diff": max_diff, "rows": rows}
+    print(f"  source check: {len(rows) - bad} of {len(rows)} fetched thumbnails match the served ones "
+          f"(diffs {[r.get('diff', r.get('error')) for r in rows]})", flush=True)
+    return bad <= 2
+
+
+async def reembed(keys, emb, storage, report):
+    """thumbnail_to_embedding for each asset from its OWN .thumbs/256x256 thumbnail (fetched from its pinned HF
+    revision), batched; the embedding and stored thumbnail replaced through the plugin's add_metadata."""
+
+    batch_size = int(os.environ.get("REEMBED_BATCH") or 16)
+    concurrency = int(os.environ.get("REEMBED_CONCURRENCY") or 4)
+    sem = asyncio.Semaphore(concurrency)
+    done = {"ok": 0, "failed": []}
+
+    async def one_batch(chunk):
+        async with sem:
+            loaded = []
+            for key in chunk:
+                try:
+                    img = await asyncio.wait_for(asyncio.to_thread(fetch_thumbnail, key), timeout=TIMEOUT_S)
+                    loaded.append(emb.load_data(omni_path=key, data=[img]))
+                except Exception as exc:  # noqa: BLE001
+                    done["failed"].append({"key": key, "error": f"thumbnail: {type(exc).__name__}: {exc}"})
+            if not loaded:
+                return
+
+            async def embed():
+                batch, idx, errors = await emb.preprocess(data=loaded)
+                results = await emb.process_valid_items(batch_data=batch, indices=idx, sample_ids=list(range(len(loaded))))
+                return batch, idx, errors, results
+
+            try:
+                batch, idx, errors, results = await with_retries(f"embedding batch of {len(loaded)}", embed)
+            except Exception as exc:  # noqa: BLE001
+                done["failed"] += [{"key": d["omni_path"], "error": str(exc)} for d in loaded]
+                return
+            for i, err in errors.items():
+                done["failed"].append({"key": loaded[i]["omni_path"], "error": f"preprocess: {err}"})
+            for i in idx:
+                key = loaded[i]["omni_path"]
+                try:
+                    content = result_content({0: results[i]}, "thumbnail_to_embedding")
+                    await emb.add_metadata(path=key, content=content, storage_client=storage)
+                    done["ok"] += 1
+                except Exception as exc:  # noqa: BLE001
+                    done["failed"].append({"key": key, "error": f"write: {type(exc).__name__}: {exc}"})
+            print(f"    re-embedded {done['ok']} so far, {len(done['failed'])} failed", flush=True)
+
+    chunks = [keys[i:i + batch_size] for i in range(0, len(keys), batch_size)]
+    await asyncio.gather(*(one_batch(c) for c in chunks))
+    report["reembed"] = {"requested": len(keys), "ok": done["ok"], "failed": done["failed"][:50], "failed_count": len(done["failed"]),
+                         "batch": batch_size, "concurrency": concurrency}
+
+
+async def main(spec_path: str) -> int:
+    from plugins.thumbnail_to_embedding import ThumbnailToEmbedding
+    from plugins.thumbnail_to_vision_metadata import ThumbnailToVisionMetadata, ThumbnailToVisionMetadataConfig
+    from search_utils.storage_client.config import StorageConfig
+    from storage.src.client import NGSearchStorageClient
+
+    spec = json.load(open(spec_path))
+    index, vlm_on = spec["index"], spec["vlm"]
+    storage = await NGSearchStorageClient.get_service(storage_config=StorageConfig(), storage_client_config=None,
+                                                      search_backend_config=None)
+    emb = ThumbnailToEmbedding()
+    vlm = ThumbnailToVisionMetadata(ThumbnailToVisionMetadataConfig(vlm_agenerate_timeout=TIMEOUT_S)) if vlm_on else None
+    report = {"index": index, "vlm": vlm_on, "model": os.environ.get("USDSEARCH_VISION_METADATA_MODEL", "") if vlm_on else "",
+              "timeout_s": TIMEOUT_S, "tries": TRIES, "assets": {}}
+    reembed_keys = spec.get("reembed") or []
+    if reembed_keys:
+        if not check_sources(index, reembed_keys, report):
+            print("IMAGE_OVERRIDES_REPORT " + json.dumps(report), flush=True)
+            print("  the fetched thumbnails are not the served ones: not re-embedding", flush=True)
+            return 1
+        await reembed(reembed_keys, emb, storage, report)
+    for key, rec in spec["assets"].items():
+        rep = report["assets"].setdefault(key, {"image": rec["image"], "ok": False})
+        t0 = time.time()
+        try:
+            img = Image.open(f"/overrides/{rec['image']}")
+            img.load()
+            sq = square(img)
+            thumb = sq.resize((THUMB_PX, THUMB_PX), Image.BILINEAR)
+            rep["source_size"] = list(img.size)
+            before = digests(os_doc(index, key))
+
+            async def embed():
+                batch, idx, errors = await emb.preprocess(data=[{"thumbnails": [thumb], "omni_path": key}])
+                if errors or idx != [0]:
+                    raise RuntimeError(f"preprocess: {errors}")
+                return result_content(await emb.process_valid_items(batch_data=batch, indices=idx, sample_ids=[0]),
+                                      "thumbnail_to_embedding")
+
+            content = await with_retries("embedding", embed)
+            rep["embedding_dims"] = [len(e) for e in content["embedding"]]
+            await emb.add_metadata(path=key, content=content, storage_client=storage)
+
+            if vlm is not None:
+                vsrc = sq
+                if rec.get("vlm_image"):
+                    vimg = Image.open(f"/overrides/{rec['vlm_image']}")
+                    vimg.load()
+                    vsrc = square(vimg)
+                    rep["vlm_image"] = rec["vlm_image"]
+                vlm_img = vsrc.resize((VLM_PX, VLM_PX), Image.BILINEAR) if max(vsrc.size) > VLM_PX else vsrc
+
+                async def describe():
+                    batch, idx, errors = await vlm.preprocess(data=[{vlm.batch_data_field: [vlm_img], "omni_path": key}])
+                    if errors or idx != [0]:
+                        raise RuntimeError(f"preprocess: {errors}")
+                    return result_content(await vlm.process_valid_items(batch_data=batch, indices=idx, sample_ids=[0]),
+                                          "thumbnail_to_vision_metadata")
+
+                meta = await with_retries("vision metadata", describe)
+                fields = next(iter(meta.values()))
+                rep["vision_fields"] = sorted(f["name"] for f in fields)
+                caption = next((f for f in fields if f["name"] == "caption"), None)
+                rep["caption"] = (caption or {}).get("value_text")
+                await vlm.add_metadata(path=key, content=meta, storage_client=storage)
+
+            urllib.request.urlopen(urllib.request.Request(f"{OS_URL}/{index}/_refresh", method="POST"), timeout=120)
+            after = digests(os_doc(index, key))
+            # A re-run on the same image writes the same embedding: unchanged fields are not a failure.
+            rep["changed_fields"] = sorted(k for k in set(before) | set(after) if before.get(k) != after.get(k))
+            rep["ok"] = vlm is None or bool(rep.get("vision_fields"))
+        except Exception as exc:  # noqa: BLE001 — reported per asset
+            rep["error"] = f"{type(exc).__name__}: {exc}"
+            traceback.print_exc()
+        rep["seconds"] = round(time.time() - t0, 1)
+        print(f"  {'ok  ' if rep['ok'] else 'FAIL'} {rec['image']}: changed {rep.get('changed_fields')}; "
+              f"caption {rep.get('caption')!r}; {rep['seconds']} s", flush=True)
+    ok = all(r["ok"] for r in report["assets"].values())
+    if reembed_keys:
+        urllib.request.urlopen(urllib.request.Request(f"{OS_URL}/{index}/_refresh", method="POST"), timeout=300)
+        # every doc carries its own vector: no zero ones and (almost) no shared ones
+        # The vectors themselves are checked on the exported dump (pipeline/image_overrides.py check_dump): a live
+        # index with k-NN derived source may not show them in _source.
+        # A failed fetch fails that asset, not the run: the dump gate after export decides (>= 98 % distinct vectors).
+        if report["reembed"]["failed_count"]:
+            print(f"  WARNING: {report['reembed']['failed_count']} asset(s) not re-embedded: {report['reembed']['failed'][:5]}", flush=True)
+    print("IMAGE_OVERRIDES_REPORT " + json.dumps(report), flush=True)
+    return 0 if ok else 1
+
+
+if __name__ == "__main__":
+    sys.exit(asyncio.run(main(sys.argv[1])))

pipeline/library_media.py

@@ -166,6 +166,27 @@ Web USD's full level (``.usdhd.tar``) and the media adds' HD files (spin.hd, rtx
 ``.hd.tar``'s (``skip``, after the choice, so they are never counted missing), and under GLB off it still asks for room
 last, after the USD archives and every add. Revert: delete it, or set the variable, and restart.
 
+USD-only stamp (2026-10-16, under GLB off only): a stamp whose main archive is ``media-<stamp>.noglb.tar`` (``USD_ONLY``)
+in place of ``media-<stamp>.tar`` carries no GLB at all: no entry of its manifest lists ``files.glb``,
+``files.articulation`` or ``hd``, and none of those files is in it. ``python -m pipeline.library_media_pack
+consolidate`` packs one from a served tree: a start's base and adds as ONE stamp, the blocks the adds replaced and the
+GLB files gone, nothing left for an add to do. Its siblings are the stamp's ``.usd.tar``, ``.usdhd.tar`` and
+``.hd-noglb.tar`` (its only HD archive: streamed whatever LIBRARY_MEDIA_HD_SPLIT says; there is no ``.hd.tar``). Its
+manifest is the served one, ``usd`` blocks included: they leave it at install as any main archive's do, and each comes
+back from the ``.usd.tar`` in the place that manifest gave it in its entry (``usd_places``), so a start serves that
+manifest byte for byte. Its entries are base entries (an entry with no GLB needs no ``entry`` stub): until its block
+lands, and for good where it does not, an entry is served with its RTX media and no model. The loader takes such a
+stamp only while GLB off is in effect (``glb_off_why``); otherwise ``archive_uri`` skips it, says so, and takes the
+newest stamp with a GLB (VIEWER_FORMAT glb, an Explorer without the USD-only reader: the stamp before it, with its adds,
+served as before). Both kinds count for the newest stamp by their stamp (a GLB archive wins a tie); a USD-only stamp's
+adds are its own, ``media-<stamp>.usd-add-<n>.tar``. ARCHIVE never matches its name, so a loader without this rule
+never opens it: uploading one changes nothing until this loader runs. While it is served, an older stamp's adds in its
+run are not applied (they were folded in, or came too late): one log line names them. An entry whose block does not
+land (no room for the ``.usd.tar``, a broken stream) has no GLB to fall back on and is served with no 3D model: one
+loud line counts and names them. Revert: LIBRARY_MEDIA_USD_ONLY=off (off / none / false / 0) and restart — the stamp
+is set aside as under VIEWER_FORMAT glb, GLB off unchanged, and the newest stamp with a GLB comes back with its adds —
+or delete its objects and restart.
+
 Joint counts (2026-10-16, both GLB modes): every count of an asset's moving parts follows one rule, the joints the
 viewer drives (``user_joints``: SimReady Central's libmedia_rules.user_joints since !733, vendored; the Explorer's
 userJointCount and isUserJoint): never a mimic, a coupling or a closed loop's solved or closing joint. The archives
@@ -229,6 +250,10 @@ from .config import BUILD_DIR, Config
 
 SUBDIR = "library-media"
 ARCHIVE = re.compile(r"^media-\d{8}T\d{4}Z\.tar$")   # media-20261005T1010Z.tar: the name sorts by time
+# A USD-only stamp's main archive (see the module doc), taken only while GLB off is in effect; ARCHIVE never matches it.
+USD_ONLY_SUFFIX = ".noglb.tar"
+USD_ONLY = re.compile(r"^media-\d{8}T\d{4}Z\.noglb\.tar$")
+STAMP_LEN = len("media-20261005T1010Z")
 # The Explorer's index key for this Space (/config.js in nginx.conf.template): the manifest's URL names it.
 INDEX_KEY = "usdsearch-dev"
 MEDIA_ROOT = BUILD_DIR / "library_media"
@@ -311,18 +336,68 @@ def run_prefix(cfg: Config, store, corpus: str) -> str:
     return json.loads(pointer)["prefix"].rstrip("/")
 
 
-def archive_uri(cfg: Config, store, prefix: str) -> Optional[str]:
+USD_ONLY_NOT_ASKED = "LIBRARY_MEDIA_GLB is not off"
+# A USD-only stamp's own switch (off / none / false / 0): set aside like an older Explorer's, with GLB off unchanged —
+# the rollback that needs no S3 delete (see the module doc).
+USD_ONLY_SWITCH = "LIBRARY_MEDIA_USD_ONLY"
+USD_ONLY_SWITCHED_OFF = f"{USD_ONLY_SWITCH} is off"
+ADDS_SHOWN = 8          # older stamps' adds a USD-only stamp leaves aside, named in a log line
+OLDER_ADD = re.compile(r"^media-\d{8}T\d{4}Z" + USD_ADD + "$")
+
+
+def archive_uri(cfg: Config, store, prefix: str, usd_only_why: Optional[str] = USD_ONLY_NOT_ASKED) -> Optional[str]:
     """The newest archive of the run at ``prefix``, else of the latest earlier run that has any; None: there is none.
-    Run ids start with the date (hierarchy/store.py), so they sort by age."""
+    Run ids start with the date (hierarchy/store.py), so they sort by age. A USD-only stamp's ``.noglb.tar`` (see the
+    module doc) counts only when ``usd_only_why`` is None (GLB off in effect), by its stamp, a GLB archive winning a
+    tie; else it is skipped, and the newest one newer than what is taken is named in a log line with ``usd_only_why``."""
     base, run = prefix.rsplit("/", 1)
     earlier = [r for r in sorted(store.list_subprefixes(cfg, base), reverse=True) if r < run]
+    skipped: Optional[tuple[str, str]] = None          # (run, name) of the newest USD-only stamp skipped
     for r in [run] + earlier:
-        names = [rel for rel, _ in store.list_objects(cfg, f"{base}/{r}/{SUBDIR}") if ARCHIVE.match(rel)]
+        rels = [rel for rel, _ in store.list_objects(cfg, f"{base}/{r}/{SUBDIR}")]
+        names = [rel for rel in rels if ARCHIVE.match(rel)]
+        only = [rel for rel in rels if USD_ONLY.match(rel)]
+        if usd_only_why is None:
+            names += only
+        elif only and skipped is None:
+            skipped = (r, max(only))
         if names:
-            return f"{base}/{r}/{SUBDIR}/{max(names)}"
+            pick = max(names, key=lambda n: (n[:STAMP_LEN], ARCHIVE.match(n) is not None))
+            if skipped is not None and (skipped[0] > r or skipped[1][:STAMP_LEN] > pick[:STAMP_LEN]):
+                print(f"==> library media: {base}/{skipped[0]}/{SUBDIR}/{skipped[1]} skipped — "
+                      f"{usd_only_set_aside(usd_only_why)}; {base}/{r}/{SUBDIR}/{pick} instead")
+            if USD_ONLY.match(pick):
+                # an older stamp's adds never apply to a newer stamp: said once, so one uploaded too late is seen
+                aside = sorted(rel for rel in rels if OLDER_ADD.match(rel) and rel[:STAMP_LEN] < pick[:STAMP_LEN])
+                if aside:
+                    print(f"==> library media: a USD-only stamp ({pick}): {len(aside)} adds of older stamps not "
+                          f"applied (only this stamp's own adds are): " + ", ".join(aside[:ADDS_SHOWN])
+                          + (f", +{len(aside) - ADDS_SHOWN} more" if len(aside) > ADDS_SHOWN else ""))
+            return f"{base}/{r}/{SUBDIR}/{pick}"
+    if skipped is not None:
+        print(f"==> library media: {base}/{skipped[0]}/{SUBDIR}/{skipped[1]} skipped — "
+              f"{usd_only_set_aside(usd_only_why)}; no archive with a GLB")
     return None
 
 
+def usd_only_set_aside(why: str) -> str:
+    """Why a USD-only stamp is skipped, for ``archive_uri``'s log line."""
+    if why == USD_ONLY_SWITCHED_OFF:
+        return f"a USD-only stamp (no GLB) set aside ({why})"
+    return f"a USD-only stamp (no GLB) needs LIBRARY_MEDIA_GLB=off in effect ({why})"
+
+
+def usd_only_stamp(uri: str) -> bool:
+    """Whether ``uri`` is a USD-only stamp's main archive (media-<stamp>.noglb.tar)."""
+    return USD_ONLY.match(uri.rsplit("/", 1)[-1]) is not None
+
+
+def stamp_base(uri: str) -> str:
+    """A stamp's main archive ``uri`` (media-<stamp>.tar or media-<stamp>.noglb.tar) less its suffix: every sibling's
+    name is this plus its own (.hd.tar, .hd-noglb.tar, .usd.tar, .usdhd.tar, .usd-add-<n>.tar, …)."""
+    return uri[: -len(USD_ONLY_SUFFIX)] if usd_only_stamp(uri) else uri[: -len(".tar")]
+
+
 def valid_manifest(path: Path) -> int | None:
     """The number of assets a manifest lists, or None when it is not one the Explorer reads (version 1, assets{})."""
     try:
@@ -930,16 +1005,19 @@ def landed(tree: Path, entry_id: str, names: set[str]) -> bool:
     return bool(names) and all((tree / entry_id / n).is_file() for n in names)
 
 
-def publish_usd(tree: Path, blocks: dict[str, dict], created: Optional[dict[str, dict]] = None) -> int:
+def publish_usd(tree: Path, blocks: dict[str, dict], created: Optional[dict[str, dict]] = None,
+                place: Optional[dict[str, list]] = None) -> int:
     """Rewrite the served manifest with exactly ``blocks`` ({Space key: usd block}) as its entries' ``usd`` (every
     other block untouched; an entry not in ``blocks`` loses any ``usd``), atomically. The number of entries carrying
     one. An unchanged manifest is not written. ``created`` (an add's new entries, {Space key: {id, moving,
     robot?}}): each one in ``blocks`` is served as exactly that plus its ``usd`` (added when absent) and the RTX media
     an add attached to it (MEDIA_KINDS, kept as served); each one not in ``blocks`` leaves the manifest whole (it has
-    nothing but its USD and those)."""
+    nothing but its USD and those). ``place`` (a USD-only stamp's ``usd_places``: {Space key: its entry's keys in the
+    stamp's manifest}): a block added where the entry had none takes the place that list gives it (keys the list does
+    not name after it, in their order); else it goes last."""
     path = tree / "manifest.json"
     doc = json.loads(path.read_text())
-    created = created or {}
+    created, place = created or {}, place or {}
     assets = doc.get("assets") if isinstance(doc.get("assets"), dict) else {}
     changed = n = 0
     for key, e in assets.items():
@@ -947,7 +1025,11 @@ def publish_usd(tree: Path, blocks: dict[str, dict], created: Optional[dict[str,
             continue
         if key in blocks:
             changed += e.get("usd") != blocks[key]
+            had = "usd" in e
             e["usd"] = blocks[key]
+            if not had and key in place:                  # a value replaced while iterating: the dict keeps its size
+                assets[key] = {k: e[k] for k in [*[k for k in place[key] if k in e], *[k for k in e
+                                                                                     if k not in place[key]]]}
             n += 1
         elif "usd" in e:
             del e["usd"]
@@ -1012,7 +1094,7 @@ def fetch_hd(cfg: Config, store, uri: str, tree: Path, not_written: int = 0,
     MediaAdds.files) is never written, whatever the manifest lists. A manifest that lists no tier-2 model streams the
     ``.hd-noglb.tar`` instead where there is one (``hd_noglb``, ``fetch_hd_noglb``; decided on the manifest, before
     ``skip``)."""
-    hd_uri = uri[: -len(".tar")] + HD_SUFFIX
+    hd_uri = stamp_base(uri) + HD_SUFFIX
     if cfg.get("LIBRARY_MEDIA_HD", "").strip().lower() in DISABLED:
         print("==> library media HD: off (LIBRARY_MEDIA_HD)")
         return 0
@@ -1024,6 +1106,13 @@ def fetch_hd(cfg: Config, store, uri: str, tree: Path, not_written: int = 0,
     allowed = {i: names for i, names in allowed.items() if names}
     if not allowed:
         return 0
+    if usd_only_stamp(uri):                               # its only HD archive (see the module doc)
+        only_uri = stamp_base(uri) + HD_NOGLB_SUFFIX
+        only_size = listed_size(cfg, store, only_uri)
+        if only_size is None:
+            print(f"==> library media HD: none at {only_uri}")
+            return 0
+        return fetch_hd_noglb(cfg, store, only_uri, only_size, tree, allowed, usd_only=True)
     noglb = hd_noglb(cfg, store, uri, manifest)
     del manifest
     if noglb is not None:
@@ -1056,7 +1145,7 @@ def hd_noglb(cfg: Config, store, uri: str, manifest: dict) -> Optional[tuple[str
     LIBRARY_MEDIA_HD_SPLIT is not off; else None, and the ``.hd.tar`` streams as before."""
     if hd_files(manifest) != hd_files(manifest, models=False):
         return None                                       # an entry lists model_hd.glb: only the .hd.tar has it
-    noglb = uri[: -len(".tar")] + HD_NOGLB_SUFFIX
+    noglb = stamp_base(uri) + HD_NOGLB_SUFFIX
     if cfg.get("LIBRARY_MEDIA_HD_SPLIT", "").strip().lower() in DISABLED:
         print(f"==> library media HD: {noglb.rsplit('/', 1)[-1]} off (LIBRARY_MEDIA_HD_SPLIT) — the .hd.tar")
         return None
@@ -1067,10 +1156,12 @@ def hd_noglb(cfg: Config, store, uri: str, manifest: dict) -> Optional[tuple[str
     return noglb, size
 
 
-def fetch_hd_noglb(cfg: Config, store, uri: str, size: int, tree: Path, allowed: dict[str, set[str]]) -> int:
+def fetch_hd_noglb(cfg: Config, store, uri: str, size: int, tree: Path, allowed: dict[str, set[str]],
+                   usd_only: bool = False) -> int:
     """Stream the ``.hd-noglb.tar`` ``uri`` (``size`` listed) into ``tree``, as ``fetch_hd`` streams the ``.hd.tar``:
     only members ``allowed`` lists, only with room for ``size``, a broken stream started again once. Returns the files
-    written; logs the listed files still missing (the Explorer stays on tier 1 for those)."""
+    written; logs the listed files still missing (the Explorer stays on tier 1 for those). ``usd_only``: a USD-only
+    stamp's, its only HD archive (said so in the log)."""
     if not room_for(tree, size, "HD (the archive without model_hd.glb)"):
         return 0
     t0 = time.time()
@@ -1080,7 +1171,8 @@ def fetch_hd_noglb(cfg: Config, store, uri: str, size: int, tree: Path, allowed:
     written, nbytes, skipped = got
     missing = sum(1 for i, names in allowed.items() for n in names if not (tree / i / n).is_file())
     print(f"==> library media HD: {written} files ({nbytes / 1e6:.0f} MB) from {uri} in {time.time() - t0:.0f} s, "
-          f"in place of the .hd.tar: the served manifest lists no tier-2 model"
+          + ("the USD-only stamp's HD archive" if usd_only else
+             "in place of the .hd.tar: the served manifest lists no tier-2 model")
           + (f"; {skipped} members not in the manifest skipped" if skipped else "")
           + (f"; {missing} listed HD files not in it (those stay on tier 1)" if missing else ""))
     return written
@@ -1092,11 +1184,12 @@ def viewer_format(cfg: Config) -> str:
     return value if value in VIEWER_FORMATS else "glb"
 
 
-def fetch_usd(cfg: Config, store, uri: str, tree: Path) -> int:
+def fetch_usd(cfg: Config, store, uri: str, tree: Path, place: Optional[dict[str, list]] = None) -> int:
     """Web USD (see the module doc): stream ``uri``'s ``.usd.tar`` sibling into the installed ``tree`` and publish the
     blocks whose tier-1 files landed (without their full level), then its ``.usdhd.tar`` and give back the full level
     of the blocks whose files landed. Returns the number of entries the served manifest then gives a ``usd``. Off
-    unless VIEWER_FORMAT is usd or auto; each archive only with room to spare (``room_for``)."""
+    unless VIEWER_FORMAT is usd or auto; each archive only with room to spare (``room_for``). ``place``: a USD-only
+    stamp's ``usd_places`` (``publish_usd``)."""
     fmt = viewer_format(cfg)
     if fmt == "glb":
         print("==> library media USD: off (VIEWER_FORMAT glb)")
@@ -1104,7 +1197,7 @@ def fetch_usd(cfg: Config, store, uri: str, tree: Path) -> int:
     if cfg.get("LIBRARY_MEDIA_USD", "").strip().lower() in DISABLED:
         print("==> library media USD: off (LIBRARY_MEDIA_USD)")
         return 0
-    base = uri[: -len(".tar")]
+    base = stamp_base(uri)
     usd_uri = base + USD_SUFFIX
     size = listed_size(cfg, store, usd_uri)
     if size is None:
@@ -1120,7 +1213,7 @@ def fetch_usd(cfg: Config, store, uri: str, tree: Path) -> int:
         return 0
     blocks, left, written, nbytes, skipped = got
     tier1 = {k: usd_without_hd(b) for k, (i, b) in blocks.items() if landed(tree, i, usd_tier1_files(b))}
-    shown = publish_usd(tree, tier1)
+    shown = publish_usd(tree, tier1, place=place)
     print(f"==> library media USD: {shown} of {len(blocks)} blocks served at tier 1 ({written} files, "
           f"{nbytes / 1e6:.0f} MB) from {usd_uri} in {time.time() - t0:.0f} s"
           + (f"; {left} blocks not in this manifest (or malformed) left out" if left else "")
@@ -1135,10 +1228,18 @@ def fetch_usd(cfg: Config, store, uri: str, tree: Path) -> int:
               file=sys.stderr)
         return shown
     if landed_hd:
-        publish_usd(tree, {**tier1, **{k: blocks[k][1] for k in landed_hd}})
+        publish_usd(tree, {**tier1, **{k: blocks[k][1] for k in landed_hd}}, place=place)
     return shown
 
 
+def usd_places(tree: Path) -> dict[str, list]:
+    """A USD-only stamp's main manifest as unpacked under ``tree``, before its ``usd`` blocks leave it: {Space key:
+    that entry's keys in order} for every entry that carries one (``publish_usd``'s ``place``)."""
+    assets = json.loads((tree / "manifest.json").read_text()).get("assets")
+    return {k: list(e) for k, e in (assets.items() if isinstance(assets, dict) else ())
+            if isinstance(e, dict) and "usd" in e}
+
+
 def fetch_usd_hd(cfg: Config, store, uri: str, tree: Path, blocks: dict[str, tuple[str, dict]],
                  what: str = "USD HD", media: Optional[dict[str, set[str]]] = None) -> set[str]:
     """Stream the ``.usdhd.tar`` ``uri`` (or an add's ``.usdhd-add-<n>.tar``; ``what`` names it in the log) into
@@ -1183,8 +1284,8 @@ def served_usd(tree: Path) -> dict[str, dict]:
 
 def usd_adds(cfg: Config, store, uri: str) -> list[tuple[int, str, int]]:
     """The supplementary web USD archives of ``uri``'s stamp (media-<stamp>.usd-add-<n>.tar): [(n, uri, size)] by n."""
-    folder, name = uri.rsplit("/", 1)
-    pattern = re.compile(re.escape(name[: -len(".tar")]) + USD_ADD)
+    folder, name = stamp_base(uri).rsplit("/", 1)
+    pattern = re.compile(re.escape(name) + USD_ADD)
     found = [(int(m.group(1)), f"{folder}/{rel}", size) for rel, size in store.list_objects(cfg, folder)
              if (m := pattern.fullmatch(rel))]
     return sorted(found)
@@ -2405,29 +2506,62 @@ def restore_glb(cfg: Config, store, uri: str, tree: Path, held: GlbHeld, ids: se
     return sum(missed.values())
 
 
-def fetch_usd_only(cfg: Config, store, uri: str, tree: Path, held: Optional[GlbHeld] = None) -> None:
+def usd_only_without_model(tree: Path, place: dict[str, list]) -> list[str]:
+    """A USD-only stamp's entries (``place``: those its manifest gave a ``usd`` block) the served manifest under
+    ``tree`` lists with no ``usd`` now: served with no 3D model at all. Their ids."""
+    assets = json.loads((tree / "manifest.json").read_text()).get("assets")
+    assets = assets if isinstance(assets, dict) else {}
+    return [str(assets[k].get("id")) for k in place if isinstance(assets.get(k), dict) and "usd" not in assets[k]]
+
+
+def lists_glb(tree: Path) -> bool:
+    """Whether an entry of the served manifest under ``tree`` lists a GLB (``files.glb`` / ``files.articulation``) or a
+    tier-2 model (``hd``)."""
+    assets = json.loads((tree / "manifest.json").read_text()).get("assets")
+    return any(isinstance(e, dict) and (e.get("hd") is not None or (isinstance(e.get("files"), dict) and any(
+        e["files"].get(r) for r in GLB_ROLES))) for e in (assets.values() if isinstance(assets, dict) else ()))
+
+
+def fetch_usd_only(cfg: Config, store, uri: str, tree: Path, held: Optional[GlbHeld] = None,
+                   place: Optional[dict[str, list]] = None) -> None:
     """LIBRARY_MEDIA_GLB=off (``glb_off_why``): the web USD archives and adds first (the GLB stays served meanwhile, but
     for the GLB skip's folders, ``held``), then ``recount_moving``, then ``drop_glb``, then the held folders kept on
     their GLB get it (``restore_glb``), then the GLB HD archive. Each step fails on its own, like ``fetch``'s; a failed
     drop_glb leaves every entry on its GLB (the manifest is written whole or not at all; every held folder restored)
     and its model_hd.glb then written as before; drop_glb removes no file. The HD room check asks for the archive less
     the model_hd.glb drop_glb unlisted: they stream through, never written. The adds may create new entries
-    (``NewEntries``): only here. The GLB HD archive never writes a file of the adds' RTX media."""
+    (``NewEntries``): only here. The GLB HD archive never writes a file of the adds' RTX media. A USD-only stamp
+    (``place``, its ``usd_places``): its blocks back in their places, and no ``drop_glb`` while no entry lists a GLB."""
     not_written = 0
     new = NewEntries()
+    only = usd_only_stamp(uri)
     try:
-        fetch_usd(cfg, store, uri, tree)
+        fetch_usd(cfg, store, uri, tree, place)
     except Exception as exc:  # noqa: BLE001 — the GLB tiers are served already; web USD is a bonus
-        print(f"==> library media USD: stopped ({type(exc).__name__}: {str(exc)[:300]}) — the GLB stays",
-              file=sys.stderr)
+        print(f"==> library media USD: stopped ({type(exc).__name__}: {str(exc)[:300]}) — "
+              + ("a USD-only stamp has no GLB to fall back on" if only else "the GLB stays"), file=sys.stderr)
     try:
         fetch_usd_adds(cfg, store, uri, tree, new)
     except Exception as exc:  # noqa: BLE001 — as above
         print(f"==> library media USD adds: stopped ({type(exc).__name__}: {str(exc)[:300]})", file=sys.stderr)
+    if only and place:
+        try:
+            no_model = usd_only_without_model(tree, place)
+        except Exception as exc:  # noqa: BLE001 — a log line only
+            no_model = [f"not counted ({type(exc).__name__})"]
+        if no_model:
+            print(f"==> library media USD: a USD-only stamp: {len(no_model)} of {len(place)} entries served with NO "
+                  f"3D model (their web USD did not land; no GLB to fall back on): " + ", ".join(no_model[:ADDS_SHOWN])
+                  + (f", +{len(no_model) - ADDS_SHOWN} more" if len(no_model) > ADDS_SHOWN else "")
+                  + f" — revert: {USD_ONLY_SWITCH}=off and restart (the stamp before it, with its adds)",
+                  file=sys.stderr)
     recount(tree, held, new.aliases)
     restore: set[str] = set()
     try:
-        _, not_written, restore = drop_glb(tree, held, new.aliases)
+        if usd_only_stamp(uri) and not lists_glb(tree):
+            print("==> library media GLB: off — a USD-only stamp: no entry lists a GLB or a model_hd.glb")
+        else:
+            _, not_written, restore = drop_glb(tree, held, new.aliases)
     except Exception as exc:  # noqa: BLE001 — every entry keeps its GLB
         restore = set(held.names) if held is not None else set()
         print(f"==> library media GLB: stopped ({type(exc).__name__}: {str(exc)[:300]}) — every entry keeps its GLB"
@@ -2462,16 +2596,23 @@ def fetch(env: dict[str, str], root: Path = MEDIA_ROOT, store=None, explorer: Op
     cfg = Config(values=dict(env))
     t0 = time.time()
     prefix = run_prefix(cfg, store, corpus)
-    uri = archive_uri(cfg, store, prefix)
+    off_why = glb_off_why(cfg, EXPLORER_ROOT if explorer is None else explorer)
+    # a USD-only stamp only while GLB off is in effect and its own switch is not off (see the module doc)
+    usd_only_why = None if off_why is None else (off_why or USD_ONLY_NOT_ASKED)
+    if usd_only_why is None and cfg.get(USD_ONLY_SWITCH, "").strip().lower() in DISABLED:
+        usd_only_why = USD_ONLY_SWITCHED_OFF
+    uri = archive_uri(cfg, store, prefix, usd_only_why)
     if uri is None:
         print(f"==> library media: none for {prefix} or an earlier run — none served")
         return 0
-    off_why = glb_off_why(cfg, EXPLORER_ROOT if explorer is None else explorer)
+    only = usd_only_stamp(uri)
     # the GLB skip: under GLB off in effect, the GLB tier-1 files are not written at install (see the module doc)
-    held = GlbHeld() if off_why is None and cfg.get(GLB_SKIP, "").strip().lower() not in DISABLED else None
+    held = (GlbHeld() if off_why is None and not only and cfg.get(GLB_SKIP, "").strip().lower() not in DISABLED
+            else None)
     root.mkdir(parents=True, exist_ok=True)
     tmp = root / f".{INDEX_KEY}.download"
     shutil.rmtree(tmp, ignore_errors=True)
+    place: dict[str, list] = {}
     try:
         archive = tmp / "media.tar"
         if not store.download_file(cfg, uri, archive):
@@ -2483,7 +2624,10 @@ def fetch(env: dict[str, str], root: Path = MEDIA_ROOT, store=None, explorer: Op
             print(f"==> library media: {uri} has no usable manifest.json — none served")
             return 0
         if '"usd"' in (tmp / "tree" / "manifest.json").read_text():
-            # web USD comes only from its own archives (fetch_usd), never from this one's manifest
+            # web USD comes only from its own archives (fetch_usd), never from this one's manifest; a USD-only
+            # stamp's blocks come back in the places its manifest gives them
+            if only:
+                place = usd_places(tmp / "tree")
             publish_usd(tmp / "tree", {})
         if held is not None:
             held.unlist(tmp / "tree")
@@ -2494,11 +2638,12 @@ def fetch(env: dict[str, str], root: Path = MEDIA_ROOT, store=None, explorer: Op
     nheld, held_bytes = held.files() if held is not None else (0, 0)
     print(f"==> library media: {count} assets from {uri} ({size / 1e6:.0f} MB) in {time.time() - t0:.0f} s"
           + (f"; GLB off: {nheld} model.glb / articulation.json ({held_bytes / 1e6:.0f} MB) not written, only an entry "
-             f"kept on its GLB gets them" if held is not None else ""))
+             f"kept on its GLB gets them" if held is not None else "")
+          + (f"; a USD-only stamp: {len(place)} web USD blocks to come from its .usd.tar" if only else ""))
     if off_why:
         print(f"==> library media GLB: off asked, but {off_why} — every entry keeps its GLB")
     if off_why is None:
-        fetch_usd_only(cfg, store, uri, root / INDEX_KEY, held)
+        fetch_usd_only(cfg, store, uri, root / INDEX_KEY, held, place)
         return count
     try:
         fetch_hd(cfg, store, uri, root / INDEX_KEY)

pipeline/library_media_pack.py

@@ -0,0 +1,522 @@
+"""The packing side of pipeline/library_media.py's archives: the ``.hd-noglb.tar``, and a USD-only stamp. Run where
+the media are built or stored (a lab pod, a one-off Action), never on a Space.
+
+    python -m pipeline.library_media_pack hd-noglb --manifest <manifest.json | media-<stamp>.tar> \\
+        [--in <media-<stamp>.hd.tar> | -] [--out <media-<stamp>.hd-noglb.tar> | -] [--summary <file.json>] \\
+        [--require-complete]
+    python -m pipeline.library_media_pack consolidate --tree <served tree> --stamp <YYYYMMDDTHHMMZ> --out <dir> \\
+        [--part-size N] [--jobs N] [--hash-only] [--drop-orphans] [--run <run id>] [--sources <listing.json>]
+
+A stamp's ``.hd.tar`` carries the tier-2 models (model_hd.glb) beside the spin HD frames and RTX clip HD files; its
+``.hd-noglb.tar`` carries those two alone (library_media's module doc, "HD without the models"). This reads the
+``.hd.tar`` once, as a stream, and writes every member ``library_media.hd_files(manifest, models=False)`` lists (a
+regular file under ``<id>/<path>``: ``library_media.member_listed``, the loader's own rule), header fields (name, size,
+mtime, mode, owner), bytes and order unchanged (``tarfile`` re-encodes the headers as pax, so the archive is not a
+byte-for-byte cut of the source); everything else (model_hd.glb, directories, members the manifest does not list) is
+left out. Streamed by the loader under a manifest that lists no tier-2 model, it writes exactly the files the
+``.hd.tar`` would.
+
+``--manifest``: the stamp's manifest, a ``manifest.json`` or the ``media-<stamp>.tar`` whose ``manifest.json`` member
+is read. ``--out`` is never written over. The summary (JSON; on stderr unless ``--summary``): members kept and left
+out by kind with their bytes, the source's and the output's size and sha256, and the listed files the source lacks
+(``--require-complete``: exit 2 when there is any). A failure leaves no output file; on stdout it leaves no
+end-of-archive marker (a reader sees a truncated archive), so a caller storing the stream must check its size and
+sha256 against the summary before keeping it.
+
+``consolidate`` (library_media's module doc, "USD-only stamp"): the tree a Space serves under GLB off — its
+``.build/library_media/usdsearch-dev/`` after a start, every entry on its web USD, base and adds applied — packed as
+ONE stamp the loader starts from alone: ``media-<stamp>.noglb.tar`` (the served ``manifest.json`` first, ``usd`` blocks
+included, then every file below that no other archive carries), ``.usd.tar`` (``usd-manifest.json`` first: each served
+block under its key and id, then each block's ``usd_tier1_files``), ``.usdhd.tar`` (each served block's
+``usd_hd_files``: only a block served at its full level names them) and ``.hd-noglb.tar`` (the files
+``hd_files(manifest, models=False)`` lists that are there: spin HD frames, RTX clip HD files). Every file of the tree
+goes in exactly one archive, so a start of the stamp alone writes the same tree and serves the same manifest byte for
+byte. Refused, before anything is written: a manifest that is not one, an entry that lists a GLB (``files.glb``,
+``files.articulation``) or a tier-2 model (``hd``: an entry kept on its GLB cannot be USD-only), a block the loader
+would not place (``usd_block_ok``) or whose files are not all there, anything but regular files, a file the loader
+would never write back (a name ``member_path`` cannot read, a dot-file, a ``.part``), a GLB file (``model.glb``,
+``model_hd.glb``, ``articulation.json`` in an id folder) no served block names. ``--drop-orphans`` leaves out the
+folders of ids no entry carries (never served; listed in the summary either way). ``--sources``: the run's
+``library-media/`` listing the tree was served from (``{"run", "objects": {name: {size, sha256?}}}``), refused when a
+name already uses the stamp or a stamp in it is not older (the loader serves only the newest), and recorded in
+pack.json with the stamps and the adds folded, so the upload can refuse a run that changed since (an add uploaded after
+the pack would be left aside). pack.json also records the Python that packed it. Deterministic: the same tree and
+stamp give the same bytes — members sorted by path after the first, pax headers with mode 0644, owner 0, no user or
+group names, mtime the stamp's own time (a fixed value, and nginx's ETag still changes from one stamp to the next).
+Each archive is written as ``<dir>/<kind>.tar.part-NNN`` parts of at most ``--part-size`` bytes (default 1,900 MiB:
+under GitHub's 2 GiB release asset cap) with ``pack.json`` (names, sizes, sha256 per archive and part, counts) and
+``SHA256SUMS``; ``--hash-only`` writes no part, only ``pack.json`` with the sizes and sha256s (a determinism check).
+``--out`` must not exist; a failure removes it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import calendar
+import hashlib
+import json
+import os
+import platform
+import re
+import shutil
+import stat
+import sys
+import tarfile
+import time
+from concurrent.futures import ProcessPoolExecutor
+from pathlib import Path
+from typing import BinaryIO, Optional
+
+from . import library_media as lm
+
+BUFSIZE = 1 << 20
+MISSING_SHOWN = 20
+PART_SIZE = 1900 << 20          # 1,992,294,400 B: every part under GitHub's 2 GiB release asset cap
+STAMP = re.compile(r"^\d{8}T\d{4}Z$")
+KINDS = ("main", "usd", "usdhd", "hd")
+SUFFIX = {"main": lm.USD_ONLY_SUFFIX, "usd": lm.USD_SUFFIX, "usdhd": lm.USD_HD_SUFFIX, "hd": lm.HD_NOGLB_SUFFIX}
+PART = {"main": "noglb.tar", "usd": "usd.tar", "usdhd": "usdhd.tar", "hd": "hd-noglb.tar"}
+# A GLB tier's file names in an id folder: a USD-only stamp carries none (one no served block names is refused).
+GLB_FILES = {"model.glb", "model_hd.glb", "articulation.json"}
+# A stamp's main archive, either kind, in a run's listing (``--sources``).
+MAIN_ARCHIVE = re.compile(r"^media-(\d{8}T\d{4}Z)(\.noglb)?\.tar$")
+ADD_OBJECT = re.compile(r"^media-\d{8}T\d{4}Z\.usd(hd)?-add-[1-9][0-9]{0,2}\.tar$")
+
+
+class Counted:
+    """A binary stream that hashes (sha256) and counts every byte read from or written to ``raw``."""
+
+    def __init__(self, raw):
+        self.raw, self.sha, self.size = raw, hashlib.sha256(), 0
+
+    def read(self, n: int = -1) -> bytes:
+        b = self.raw.read(n) if n is not None and n >= 0 else self.raw.read()
+        self.sha.update(b)
+        self.size += len(b)
+        return b
+
+    def write(self, b) -> int:
+        self.raw.write(b)
+        self.sha.update(b)
+        self.size += len(b)
+        return len(b)
+
+    def flush(self) -> None:
+        if hasattr(self.raw, "flush"):
+            self.raw.flush()
+
+    def drain(self) -> None:
+        """Read the rest (a tar's end blocks and record padding), so ``size`` and ``sha`` cover the whole source."""
+        for _ in iter(lambda: self.read(BUFSIZE), b""):
+            pass
+
+    def facts(self) -> dict:
+        return {"bytes": self.size, "sha256": self.sha.hexdigest()}
+
+
+def split_hd(src: BinaryIO, out: BinaryIO, manifest: dict) -> dict:
+    """Write to ``out`` the ``.hd-noglb.tar`` of the ``.hd.tar`` read from ``src`` (once, in order) for ``manifest``.
+    Returns the summary (the module doc)."""
+    keep = lm.hd_files(manifest, models=False)
+    models = lm.hd_files(manifest)
+    reader, writer = Counted(src), Counted(out)
+    kept = {"spin_hd": [0, 0], "clip_hd": [0, 0]}
+    left = {"model_hd": [0, 0], "unlisted": [0, 0], "directory": [0, 0], "other": [0, 0]}
+    seen: set[tuple[str, str]] = set()
+    duplicates = 0
+    with tarfile.open(fileobj=reader, mode="r|", bufsize=BUFSIZE) as tin, \
+            tarfile.open(fileobj=writer, mode="w|", bufsize=BUFSIZE) as tout:
+        for m in tin:
+            where = lm.member_path(m.name)
+            if lm.member_listed(m, keep):
+                with tin.extractfile(m) as data:
+                    tout.addfile(m, data)
+                kind = "spin_hd" if where[1].startswith("spin_hd_") else "clip_hd"
+                kept[kind][0] += 1
+                kept[kind][1] += m.size
+                duplicates += where in seen
+                seen.add(where)
+                continue
+            if m.isdir():
+                kind = "directory"
+            elif not m.isfile():
+                kind = "other"
+            elif where is not None and where[1] in models.get(where[0], ()):
+                kind = "model_hd"
+            else:
+                kind = "unlisted"
+            left[kind][0] += 1
+            left[kind][1] += m.size if m.isfile() else 0
+    reader.drain()
+    missing = sorted(f"{i}/{n}" for i, names in keep.items() for n in names if (i, n) not in seen)
+    listed = sum(len(names) for names in keep.values())
+    return {
+        "source": reader.facts(),
+        "output": {**writer.facts(), "members": sum(n for n, _ in kept.values())},
+        "kept": {k: {"members": n, "bytes": b} for k, (n, b) in kept.items()},
+        "left_out": {k: {"members": n, "bytes": b} for k, (n, b) in left.items()},
+        "entries": len(keep),
+        "entries_with_models": sum(1 for i, names in models.items() if names - keep.get(i, set())),
+        "listed": listed,
+        "listed_missing": len(missing),
+        "missing_sample": missing[:MISSING_SHOWN],
+        "duplicates": duplicates,
+    }
+
+
+def read_manifest(path: Path) -> tuple[dict, str]:
+    """The manifest at ``path`` (a manifest.json, or a media-<stamp>.tar whose manifest.json member is read as a
+    stream) and the sha256 of its bytes. Not one the Explorer reads (version 1, assets{}): SystemExit."""
+    raw = None
+    if path.suffix == ".tar":
+        with open(path, "rb") as fh, tarfile.open(fileobj=fh, mode="r|", bufsize=BUFSIZE) as tf:
+            for m in tf:
+                if m.isfile() and m.name.removeprefix("./") == "manifest.json":
+                    with tf.extractfile(m) as data:
+                        raw = data.read()
+                    break
+        if raw is None:
+            raise SystemExit(f"{path}: no manifest.json member")
+    else:
+        raw = path.read_bytes()
+    try:
+        doc = json.loads(raw)
+    except ValueError as exc:
+        raise SystemExit(f"{path}: manifest is not JSON ({exc})") from None
+    if not isinstance(doc, dict) or doc.get("version") != 1 or not isinstance(doc.get("assets"), dict):
+        raise SystemExit(f"{path}: not a library media manifest (version 1, assets{{}})")
+    return doc, hashlib.sha256(raw).hexdigest()
+
+
+def hd_noglb(a: argparse.Namespace) -> int:
+    manifest, manifest_sha = read_manifest(a.manifest)
+    out_path = None if a.out == "-" else Path(a.out)
+    src = sys.stdin.buffer if a.src == "-" else open(a.src, "rb")
+    try:
+        out = sys.stdout.buffer if out_path is None else open(out_path, "xb")   # never written over
+    except BaseException:
+        if src is not sys.stdin.buffer:
+            src.close()
+        raise
+    try:
+        summary = split_hd(src, out, manifest)
+    except BaseException:
+        if out_path is not None:
+            out.close()
+            out_path.unlink(missing_ok=True)
+        raise
+    finally:
+        if src is not sys.stdin.buffer:
+            src.close()
+    if out_path is not None:
+        out.close()
+    else:
+        out.flush()
+    summary = {"manifest_sha256": manifest_sha, "manifest_entries": len(manifest["assets"]), **summary}
+    text = json.dumps(summary, indent=1)
+    if a.summary:
+        a.summary.write_text(text + "\n")
+    else:
+        print(text, file=sys.stderr)
+    if a.require_complete and summary["listed_missing"]:
+        print(f"{summary['listed_missing']} listed HD files are not in the source "
+              f"(e.g. {', '.join(summary['missing_sample'][:3])})", file=sys.stderr)
+        return 2
+    return 0
+
+
+# --- consolidate: a served tree as ONE USD-only stamp (the module doc) ------------------------------------------------
+
+class Parts:
+    """A write-only stream into ``<folder>/<name>.part-NNN`` files of at most ``limit`` bytes each (``folder`` None:
+    nothing written, only hashed), hashing (sha256) the whole stream and each part."""
+
+    def __init__(self, folder: Optional[Path], name: str, limit: int):
+        self.folder, self.name, self.limit = folder, name, limit
+        self.sha, self.size = hashlib.sha256(), 0
+        self.parts: list[dict] = []
+        self.fh = None
+
+    def write(self, b) -> int:
+        mv = memoryview(b).cast("B")
+        n = len(mv)
+        self.sha.update(mv)
+        self.size += n
+        while len(mv):
+            if not self.parts or self.parts[-1]["size"] >= self.limit:
+                self._next()
+            part = self.parts[-1]
+            take = mv[: self.limit - part["size"]]
+            part["sha"].update(take)
+            part["size"] += len(take)
+            if self.fh is not None:
+                self.fh.write(take)
+            mv = mv[len(take):]
+        return n
+
+    def _next(self) -> None:
+        if self.fh is not None:
+            self.fh.close()
+        name = f"{self.name}.part-{len(self.parts):03d}"
+        self.parts.append({"name": name, "size": 0, "sha": hashlib.sha256()})
+        self.fh = None if self.folder is None else open(self.folder / name, "xb")    # never over a file
+
+    def flush(self) -> None:
+        if self.fh is not None:
+            self.fh.flush()
+
+    def close(self) -> dict:
+        if self.fh is not None:
+            self.fh.close()
+            self.fh = None
+        return {"size": self.size, "sha256": self.sha.hexdigest(), "parts": [p["name"] for p in self.parts],
+                "part_sha256": [p["sha"].hexdigest() for p in self.parts],
+                "part_sizes": [p["size"] for p in self.parts]}
+
+
+def member_info(name: str, size: int, mtime: int) -> tarfile.TarInfo:
+    """A regular-file header with nothing of the packing host in it: mode 0644, owner 0, no names, ``mtime``."""
+    ti = tarfile.TarInfo(name)
+    ti.size, ti.mtime, ti.mode, ti.type = size, mtime, 0o644, tarfile.REGTYPE
+    ti.uid = ti.gid = 0
+    ti.uname = ti.gname = ""
+    return ti
+
+
+def plan_consolidate(tree: Path, drop_orphans: bool = False) -> dict:
+    """Which file of the served ``tree`` goes in which archive (the module doc), the overlay, and the facts; every
+    refusal at once (SystemExit), before anything is written."""
+    try:
+        raw = (tree / "manifest.json").read_bytes()
+        doc = json.loads(raw)
+    except (OSError, ValueError) as exc:
+        raise SystemExit(f"{tree}: no readable manifest.json ({exc})") from None
+    if not isinstance(doc, dict) or doc.get("version") != 1 or not isinstance(doc.get("assets"), dict):
+        raise SystemExit(f"{tree}: not a library media manifest (version 1, assets{{}})")
+    problems: list[str] = []
+    usd1: dict[str, set[str]] = {}
+    usdhd: dict[str, set[str]] = {}
+    overlay: dict[str, dict] = {}
+    ids: set[str] = set()
+    named: dict[str, set[str]] = {}                     # every string the entries of an id give (their file names)
+    full = 0
+    for key, e in doc["assets"].items():
+        if not isinstance(e, dict) or not isinstance(e.get("id"), str) or not lm.SAFE_ID.match(e["id"]):
+            problems.append(f"{str(key)[-120:]}: no safe id")
+            continue
+        i = e["id"]
+        ids.add(i)
+        lm._strings(e, named.setdefault(i, set()))
+        files = e.get("files") if isinstance(e.get("files"), dict) else {}
+        if e.get("hd") is not None or any(files.get(r) for r in lm.GLB_ROLES):
+            problems.append(f"{i}: lists a GLB or a model_hd.glb (an entry kept on its GLB is never USD-only)")
+            continue
+        if "usd" not in e:
+            continue
+        if not lm.usd_block_ok(e["usd"]):
+            problems.append(f"{i}: a usd block the loader would not place (usd_block_ok)")
+            continue
+        overlay[key] = {"id": i, "usd": e["usd"]}
+        usd1.setdefault(i, set()).update(lm.usd_tier1_files(e["usd"]))
+        hd_names = lm.usd_hd_files(e["usd"])
+        usdhd.setdefault(i, set()).update(hd_names)
+        full += bool(hd_names)
+    hd = lm.hd_files(doc, models=False)
+    groups: dict[str, list[tuple[str, int]]] = {k: [] for k in KINDS}
+    found: set[tuple[str, str]] = set()
+    orphans: dict[str, list[int]] = {}
+    top: list[str] = []
+    for dirpath, dirnames, filenames in os.walk(tree):
+        dirnames.sort()
+        for d in dirnames:
+            if os.path.islink(Path(dirpath) / d):
+                problems.append(f"{(Path(dirpath) / d).relative_to(tree).as_posix()}: not a regular folder")
+        for fn in sorted(filenames):
+            p = Path(dirpath) / fn
+            rel = p.relative_to(tree).as_posix()
+            st = os.lstat(p)
+            if not stat.S_ISREG(st.st_mode):
+                problems.append(f"{rel}: not a regular file")
+                continue
+            if any(seg.startswith(".") for seg in rel.split("/")):
+                problems.append(f"{rel}: a dot-file (a .part left over?)")
+                continue
+            if rel == "manifest.json":
+                continue
+            where = lm.member_path(rel)
+            if where is None:
+                top.append(rel)                           # unpack writes it back; nothing names it
+                groups["main"].append((rel, st.st_size))
+                continue
+            i, path = where
+            if i not in ids:
+                orphans.setdefault(i, [0, 0])
+                orphans[i][0] += 1
+                orphans[i][1] += st.st_size
+                if drop_orphans:
+                    continue
+            if path in GLB_FILES and path not in named.get(i, ()):
+                problems.append(f"{rel}: a GLB file no served block names (a USD-only stamp carries none"
+                                + ("; an orphan folder: --drop-orphans" if i not in ids else
+                                   "; a tree served with LIBRARY_MEDIA_GLB_SKIP=off?") + ")")
+                continue
+            found.add(where)
+            kind = ("usd" if path in usd1.get(i, ()) else "usdhd" if path in usdhd.get(i, ()) else
+                    "hd" if path in hd.get(i, ()) else "main")
+            groups[kind].append((rel, st.st_size))
+    for kind, listed in (("usd", usd1), ("usdhd", usdhd)):
+        gone = sorted(f"{i}/{n}" for i, names in listed.items() for n in names if (i, n) not in found)
+        if gone:
+            problems.append(f"{len(gone)} files a served block names for the {kind} archive are not there "
+                            f"(e.g. {', '.join(gone[:3])})")
+    if problems:
+        raise SystemExit(f"{tree}: not packed ({len(problems)} problems): " + "; ".join(problems[:MISSING_SHOWN])
+                         + (f"; +{len(problems) - MISSING_SHOWN} more" if len(problems) > MISSING_SHOWN else ""))
+    hd_missing = sorted(f"{i}/{n}" for i, names in hd.items() for n in names if (i, n) not in found)
+    for kind in KINDS:
+        groups[kind].sort()
+    overlay_raw = json.dumps({"version": 1, "profile": lm.USD_PROFILE, "assets": overlay},
+                             separators=(",", ":")).encode()
+    return {
+        "manifest": raw, "overlay": overlay_raw, "groups": groups,
+        "facts": {
+            "entries": len(doc["assets"]), "usd_blocks": len(overlay), "usd_full": full,
+            "hd_listed": sum(len(names) for names in hd.values()), "hd_listed_missing": len(hd_missing),
+            "hd_missing_sample": hd_missing[:MISSING_SHOWN], "top_level_files": top,
+            "orphans": {"ids": len(orphans), "files": sum(n for n, _ in orphans.values()),
+                        "bytes": sum(b for _, b in orphans.values()), "dropped": drop_orphans,
+                        "sample": sorted(orphans)[:MISSING_SHOWN]},
+            "tree": {"files": 1 + sum(len(g) for g in groups.values()),
+                     "bytes": len(raw) + sum(n for g in groups.values() for _, n in g)},
+        },
+    }
+
+
+def write_archive(tree: Path, kind: str, members: list[tuple[str, int]], first: Optional[tuple[str, bytes]],
+                  out: Optional[Path], name: str, part_size: int, mtime: int) -> dict:
+    """One archive of the stamp: ``first`` (name, bytes) then ``members`` (paths below ``tree``, with the size each
+    had when planned: a file that changed since fails it), as ``out``/<part name>.part-NNN (``out`` None: hashed
+    only). Its facts for pack.json."""
+    sink = Parts(out, PART[kind], part_size)
+    with tarfile.open(fileobj=sink, mode="w|", format=tarfile.PAX_FORMAT, bufsize=BUFSIZE) as tf:
+        if first is not None:
+            tf.addfile(member_info(first[0], len(first[1]), mtime), _Bytes(first[1]))
+        for rel, size in members:
+            with open(tree / rel, "rb") as fh:
+                if os.fstat(fh.fileno()).st_size != size:
+                    raise SystemExit(f"{tree / rel}: changed while packed ({size} B planned)")
+                tf.addfile(member_info(rel, size, mtime), fh)
+    facts = sink.close()
+    return {"name": name, **facts, "members": len(members) + (first is not None),
+            "files_bytes": sum(n for _, n in members) + (len(first[1]) if first else 0)}
+
+
+class _Bytes:
+    """A minimal reader over ``data`` for tarfile.addfile."""
+
+    def __init__(self, data: bytes):
+        self.data, self.pos = data, 0
+
+    def read(self, n: int = -1) -> bytes:
+        end = len(self.data) if n is None or n < 0 else self.pos + n
+        b = self.data[self.pos:end]
+        self.pos += len(b)
+        return b
+
+
+def read_sources(path: Path, stamp: str) -> dict:
+    """``--sources``: the run's ``library-media/`` listing when the served tree was made, ``{"run": <run id>,
+    "objects": {<name>: {"size": <bytes>, "sha256": <hex, optional>}}}`` (the folded stamp and adds among them).
+    Refused (SystemExit) when a name already uses ``stamp`` or a stamp in it is not older than ``stamp`` (the loader
+    serves only the newest). Returned as recorded in pack.json, with the main archives' stamps and the adds folded."""
+    try:
+        doc = json.loads(Path(path).read_text())
+        objects = doc["objects"]
+        if not isinstance(objects, dict) or not all(isinstance(n, str) and isinstance(o, dict)
+                                                    and isinstance(o.get("size"), int) for n, o in objects.items()):
+            raise ValueError("objects: {name: {size, sha256?}}")
+    except (OSError, ValueError, KeyError, TypeError) as exc:
+        raise SystemExit(f"--sources {path}: unreadable ({exc})") from None
+    used = sorted(n for n in objects if n.startswith(f"media-{stamp}."))
+    if used:
+        raise SystemExit(f"--stamp {stamp} is already used in the run: {', '.join(used[:MISSING_SHOWN])}")
+    stamps = sorted({m.group(1) for n in objects if (m := MAIN_ARCHIVE.match(n))})
+    if stamps and stamps[-1] >= stamp:
+        raise SystemExit(f"--stamp {stamp} is not newer than the run's newest stamp {stamps[-1]}: never served")
+    adds = sorted(n for n in objects if ADD_OBJECT.match(n))
+    return {"run": doc.get("run"), "stamps": stamps, "adds": adds,
+            "objects": {n: objects[n] for n in sorted(objects)}}
+
+
+def consolidate(a: argparse.Namespace) -> int:
+    if not STAMP.match(a.stamp):
+        raise SystemExit(f"--stamp {a.stamp!r} is not YYYYMMDDTHHMMZ")
+    if a.part_size < tarfile.BLOCKSIZE:
+        raise SystemExit(f"--part-size under {tarfile.BLOCKSIZE} B")
+    sources = read_sources(a.sources, a.stamp) if a.sources else None
+    if sources is None:
+        print("library_media_pack: no --sources — pack.json records no run listing, so the upload cannot check that "
+              "every add was folded", file=sys.stderr)
+    plan = plan_consolidate(a.tree, a.drop_orphans)
+    mtime = calendar.timegm(time.strptime(a.stamp, "%Y%m%dT%H%MZ"))
+    a.out.mkdir(parents=True)                             # never into an existing folder
+    try:
+        firsts = {"main": ("manifest.json", plan["manifest"]), "usd": (lm.USD_OVERLAY, plan["overlay"])}
+        args = [(a.tree, kind, plan["groups"][kind], firsts.get(kind), None if a.hash_only else a.out,
+                 f"media-{a.stamp}{SUFFIX[kind]}", a.part_size, mtime) for kind in KINDS]
+        if a.jobs <= 1:
+            archives = {kind: write_archive(*arg) for kind, arg in zip(KINDS, args)}
+        else:
+            with ProcessPoolExecutor(min(a.jobs, len(KINDS))) as ex:
+                archives = dict(zip(KINDS, [j.result() for j in [ex.submit(write_archive, *arg) for arg in args]]))
+        pack = {
+            "kind": "usd-only stamp", "stamp": a.stamp, "run": a.run, "profile": lm.USD_PROFILE,
+            **archives,
+            "manifest_sha256": hashlib.sha256(plan["manifest"]).hexdigest(),
+            "overlay_sha256": hashlib.sha256(plan["overlay"]).hexdigest(),
+            "mtime": mtime, "part_size": a.part_size, "hash_only": a.hash_only,
+            "loader": "pipeline/library_media.py with USD_ONLY (a USD-only stamp, under GLB off only)",
+            "python": platform.python_version(), "sources": sources,
+            **plan["facts"],
+        }
+        (a.out / "pack.json").write_text(json.dumps(pack, indent=1) + "\n")
+        if not a.hash_only:
+            sums = sorted((p, s) for k in KINDS for p, s in zip(archives[k]["parts"], archives[k]["part_sha256"]))
+            (a.out / "SHA256SUMS").write_text("".join(f"{s}  {p}\n" for p, s in sums))
+    except BaseException:
+        shutil.rmtree(a.out, ignore_errors=True)
+        raise
+    print(json.dumps({k: (pack[k] if k not in KINDS else {f: pack[k][f] for f in ("name", "size", "sha256", "members")})
+                      for k in ("stamp", *KINDS, "entries", "usd_blocks", "usd_full", "hd_listed",
+                                "hd_listed_missing", "orphans", "tree")}, indent=1))
+    return 0
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="python -m pipeline.library_media_pack",
+                                 description="Pack library media archives (see the module doc).")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    p = sub.add_parser("hd-noglb", help="a stamp's .hd-noglb.tar from its .hd.tar and manifest")
+    p.add_argument("--manifest", type=Path, required=True, help="manifest.json, or the stamp's media-<stamp>.tar")
+    p.add_argument("--in", dest="src", default="-", help="the .hd.tar (default: stdin)")
+    p.add_argument("--out", default="-", help="the .hd-noglb.tar to write, never over a file (default: stdout)")
+    p.add_argument("--summary", type=Path, help="write the summary JSON here (default: stderr)")
+    p.add_argument("--require-complete", action="store_true",
+                   help="exit 2 when a file the manifest lists is not in the source")
+    c = sub.add_parser("consolidate", help="a served tree (GLB off) as ONE USD-only stamp")
+    c.add_argument("--tree", type=Path, required=True, help="the served tree (.build/library_media/usdsearch-dev)")
+    c.add_argument("--stamp", required=True, help="the new stamp, YYYYMMDDTHHMMZ (UTC)")
+    c.add_argument("--out", type=Path, required=True, help="a folder that does not exist yet")
+    c.add_argument("--run", default=None, help="the index run it is for (recorded in pack.json)")
+    c.add_argument("--part-size", type=int, default=PART_SIZE)
+    c.add_argument("--jobs", type=int, default=4, help="archives written at once (default 4)")
+    c.add_argument("--hash-only", action="store_true", help="no part written: pack.json with sizes and sha256s")
+    c.add_argument("--drop-orphans", action="store_true", help="leave out the folders of ids no entry carries")
+    c.add_argument("--sources", type=Path, default=None,
+                   help="the run's library-media listing the tree was served from (JSON; recorded in pack.json)")
+    a = ap.parse_args(argv)
+    return hd_noglb(a) if a.cmd == "hd-noglb" else consolidate(a)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pipeline/package_download/app.py

@@ -22,7 +22,7 @@ from fastapi import FastAPI, Query, Request
 from fastapi.responses import FileResponse, JSONResponse, Response, StreamingResponse
 from starlette.background import BackgroundTask
 
-from . import bundle, telemetry
+from . import bundle, dedupe, telemetry
 from .catalog import Catalog
 from .settings import Settings
 from .sizes import PackageSizes
@@ -336,6 +336,14 @@ def create_app(
         if manifest_only:
             return JSONResponse(public(bundle.preview(asset_url, package, listed)))
 
+        shared: dict = {}
+        if settings.dedupe:
+            try:
+                shared = await dedupe.plan(package, listed, source.read)
+            except Exception as exc:  # noqa: BLE001 — best effort: every file is fetched on its own
+                logger.warning("shared fetches for %s skipped: %s", package.root_address, exc)
+        stats["n_shared"] = sum(len(s.copies) for s in shared.values())
+
         if settings.max_bundle_bytes > 0 and total > settings.max_bundle_bytes:
             raise _HTTPError(
                 413, f"Download bundle is too large ({total} bytes; limit is {settings.max_bundle_bytes})."
@@ -360,6 +368,7 @@ def create_app(
                     read_ahead_bytes=settings.read_ahead_bytes,
                     connections=connections,
                     retry=retry,
+                    shared=shared,
                 )
             except bundle.RootMissing as exc:
                 raise _HTTPError(404, f"Asset '{to_public_url(asset_url)}' could not be retrieved: {exc}")

pipeline/package_download/bundle.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 import asyncio
 import contextlib
+import hashlib
 import json
 import logging
 import math
@@ -25,7 +26,7 @@ import zipfile
 from collections import deque
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
-from typing import AsyncIterator, Awaitable, Callable, Deque, Dict, List, Optional, Tuple, TypeVar
+from typing import Any, AsyncIterator, Awaitable, Callable, Deque, Dict, List, Optional, Tuple, TypeVar
 
 from .catalog import Package
 
@@ -313,6 +314,7 @@ async def build_zip(
     read_ahead_bytes: int = 0,
     connections: Optional[asyncio.Semaphore] = None,
     retry: RetryPolicy = NO_RETRY,
+    shared: Optional[Dict[str, Any]] = None,
 ) -> BundleResult:
     """Fetch every listed file (bounded concurrency) and write it into a ZIP at ``zip_path``.
 
@@ -330,6 +332,10 @@ async def build_zip(
     backoff — a ``connections`` slot (reads in flight across every bundle). A transient
     failure is retried per ``retry`` from byte 0: nothing reaches the archive before a
     file is whole. Cancelled, it stops every read and the writer before it returns.
+
+    ``shared`` (dedupe.plan: path -> Shared(sha256, copies)) fetches a file once for every
+    path that holds the same bytes: its copies are written from its spool, after its
+    sha256 is checked. A fetch that fails or does not match fetches each copy on its own.
     """
     result = BundleResult()
     semaphore = asyncio.Semaphore(max(concurrency, 1))
@@ -339,15 +345,23 @@ async def build_zip(
     in_ram = max(min(spool_bytes, read_ahead_bytes) if read_ahead_bytes > 0 else spool_bytes, 1)
     mode = compression.strip().lower()
     started = time.perf_counter()
+    shared = shared or {}
+    copied = {c.path for s in shared.values() for c in s.copies}
+    digests: Dict[str, str] = {}  # sha256 of a shared fetch, checked before its copies are written
 
     async def download(entry: FileEntry, spool) -> int:
         async def attempt() -> int:
             spool.seek(0)
             spool.truncate()  # a retry starts over: nothing of a failed try is kept
+            digest = hashlib.sha256() if entry.path in shared else None
             async with limited(connections):
                 async with contextlib.aclosing(read(package, entry.path)) as chunks:
                     async for chunk in chunks:
                         spool.write(chunk)
+                        if digest is not None:
+                            digest.update(chunk)
+            if digest is not None:
+                digests[entry.path] = digest.hexdigest()
             return spool.tell()
 
         return await retrying(attempt, retry, package.address(entry.path))
@@ -372,37 +386,63 @@ async def build_zip(
         spool.seek(0)
         return entry, spool, size, None, cost
 
-    tasks = [asyncio.create_task(fetch(f)) for f in files]
+    tasks: List[asyncio.Task] = []
     stop = threading.Event()  # tells the writer thread to let go of the member it is copying
+
+    async def write(writer, zf, entry: FileEntry, spool, size: int) -> None:
+        address = package.address(entry.path)
+        result.total_bytes += size
+        if max_bytes > 0 and result.total_bytes > max_bytes:
+            raise BundleTooLarge(f"Download bundle exceeded the size limit of {max_bytes} bytes.")
+        arc = safe_arcname(package, entry.path)
+        if arc is None:
+            result.errors[address] = "path outside the package"
+            return
+        info = zipfile.ZipInfo(arc, date_time=time.localtime()[:6])
+        info.compress_type = _compress_type(arc, mode, stored_extensions)
+        info.external_attr = 0o644 << 16
+        await _write_member(writer, stop, zf, info, spool, size)
+        result.files.append({"url": address, "path": arc, "size": size})
+
+    async def run(writer, zf, entries: List[FileEntry]) -> List[FileEntry]:
+        """Fetch and write ``entries``; returns the copies whose shared fetch failed or did not match."""
+        batch = [asyncio.create_task(fetch(f)) for f in entries]
+        tasks.extend(batch)
+        again: List[FileEntry] = []
+        for completed in asyncio.as_completed(batch):
+            entry, spool, size, exc, cost = await completed
+            group = shared.get(entry.path)
+            if spool is None:
+                if isinstance(exc, SourceError) and exc.kind in ("denied", "busy", "unavailable"):
+                    raise exc  # the backend itself is down; a ZIP of holes helps nobody
+                if entry.path == package.asset_path:
+                    raise RootMissing(str(exc) or "not found")
+                result.errors[package.address(entry.path)] = str(exc) or type(exc).__name__
+                again.extend(group.copies if group else ())
+                continue
+            try:
+                with spool:
+                    await write(writer, zf, entry, spool, size)
+                    if group:
+                        if digests.get(entry.path) != group.sha256:
+                            logger.warning("%s does not match its BOM sha256: its copies are fetched on their own",
+                                           package.address(entry.path))
+                            again.extend(group.copies)
+                            continue
+                        for copy in group.copies:
+                            spool.seek(0)
+                            await write(writer, zf, copy, spool, size)
+            finally:
+                budget.release(cost)
+        return again
+
     try:
         with ThreadPoolExecutor(max_workers=1, thread_name_prefix="package-zip") as writer:
             with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as zf:
-                for completed in asyncio.as_completed(tasks):
-                    entry, spool, size, exc, cost = await completed
-                    address = package.address(entry.path)
-                    if spool is None:
-                        if isinstance(exc, SourceError) and exc.kind in ("denied", "busy", "unavailable"):
-                            raise exc  # the backend itself is down; a ZIP of holes helps nobody
-                        if entry.path == package.asset_path:
-                            raise RootMissing(str(exc) or "not found")
-                        result.errors[address] = str(exc) or type(exc).__name__
-                        continue
-                    try:
-                        with spool:
-                            result.total_bytes += size
-                            if max_bytes > 0 and result.total_bytes > max_bytes:
-                                raise BundleTooLarge(f"Download bundle exceeded the size limit of {max_bytes} bytes.")
-                            arc = safe_arcname(package, entry.path)
-                            if arc is None:
-                                result.errors[address] = "path outside the package"
-                                continue
-                            info = zipfile.ZipInfo(arc, date_time=time.localtime()[:6])
-                            info.compress_type = _compress_type(arc, mode, stored_extensions)
-                            info.external_attr = 0o644 << 16
-                            await _write_member(writer, stop, zf, info, spool, size)
-                            result.files.append({"url": address, "path": arc, "size": size})
-                    finally:
-                        budget.release(cost)
+                again = await run(writer, zf, [f for f in files if f.path not in copied])
+                if again:
+                    # Each on its own, as without a BOM; a copy is never itself shared.
+                    await run(writer, zf, again)
     finally:
         for task in tasks:
             task.cancel()

pipeline/package_download/dedupe.py

@@ -0,0 +1,97 @@
+"""Byte-identical files of a package, fetched from the source once.
+
+SimReady Central product review 2026-10-14 (OMPE-116021, download speed): every package in
+``nvidia/simready-assets`` ships its textures twice, under ``simready_usd/Textures/`` (what the USD
+references) and as byte-identical copies under ``simready_usd/source_assets/``: about half of a
+package's bytes (``rack_composition_l09``: 238 + 238 of 599 MB; ``robotic_system_console_b01``:
+981 + 981 of 2,026 MB). The package's BOM (``.metadata/com.nvidia.simready.packaging.bom.json``,
+every file with its sha256 and size) tells which.
+
+The download keeps every file: the BOM lists the copies, so a package without them would fail its
+own validation. What changes is how often the bytes are read: the files the BOM gives one sha256 and
+one size, and the listing that same size, are fetched once and written into the archive under every
+path that has them (``bundle.build_zip``, which checks the fetched bytes against the sha256 first and
+fetches the other paths on their own when they differ). No BOM, a malformed one, or a file it does not
+cover: fetched as before.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from typing import AsyncIterator, Callable, Dict, List, Tuple
+
+from .bundle import FileEntry
+from .catalog import Package
+
+logger = logging.getLogger("package_download")
+
+BOM_PATH = ".metadata/com.nvidia.simready.packaging.bom.json"
+SOURCE_DIR = "source_assets"
+MAX_BOM = 16 * 1024 * 1024
+
+Reader = Callable[[Package, str], AsyncIterator[bytes]]
+
+
+@dataclass
+class Shared:
+    """One fetch for several paths: the BOM's sha256 of the bytes, and the other files that hold them."""
+
+    sha256: str
+    copies: List[FileEntry] = field(default_factory=list)
+
+
+def bom_items(doc) -> Dict[str, Tuple[str, int]]:
+    """Package-relative path -> (sha256, size) from a BOM document; {} when it has none."""
+    out: Dict[str, Tuple[str, int]] = {}
+    items = doc.get("items") if isinstance(doc, dict) else None
+    for item in items if isinstance(items, list) else ():
+        if not isinstance(item, dict):
+            continue
+        path, digest, size = item.get("relative_path"), (item.get("hash") or {}).get("sha256"), item.get("size")
+        if isinstance(path, str) and isinstance(digest, str) and path and digest and isinstance(size, int):
+            out[path] = (digest.lower(), size)
+    return out
+
+
+def under_source(rel: str) -> bool:
+    return SOURCE_DIR in rel.split("/")[:-1]
+
+
+def groups(package: Package, listed: List[FileEntry], items: Dict[str, Tuple[str, int]]) -> Dict[str, Shared]:
+    """Path of the file fetched -> Shared, for every BOM sha256 that two or more listed files hold.
+
+    A file counts only when the listing's size equals the BOM's. The one fetched is the first outside
+    any ``source_assets/`` folder (what the USD references), else the first by path; the root asset is
+    always fetched on its own.
+    """
+    by_hash: Dict[Tuple[str, int], List[FileEntry]] = {}
+    for f in listed:
+        hit = items.get(package.relative(f.path))
+        if hit and hit[1] == f.size and f.path != package.asset_path:
+            by_hash.setdefault(hit, []).append(f)
+    out: Dict[str, Shared] = {}
+    for (digest, _), files in by_hash.items():
+        if len(files) < 2:
+            continue
+        files = sorted(files, key=lambda f: (under_source(package.relative(f.path)), f.path))
+        out[files[0].path] = Shared(digest, files[1:])
+    return out
+
+
+async def _read_all(read: Reader, package: Package, path: str, limit: int) -> bytes:
+    data = bytearray()
+    async for chunk in read(package, path):
+        data += chunk
+        if len(data) > limit:
+            raise ValueError(f"{path} is over {limit} bytes")
+    return bytes(data)
+
+
+async def plan(package: Package, listed: List[FileEntry], read: Reader) -> Dict[str, Shared]:
+    """The shared fetches of a package (one read: its BOM); {} without a usable BOM."""
+    bom = next((f for f in listed if package.relative(f.path) == BOM_PATH), None)
+    if bom is None or bom.size > MAX_BOM:
+        return {}
+    return groups(package, listed, bom_items(json.loads(await _read_all(read, package, bom.path, MAX_BOM))))

pipeline/package_download/settings.py

@@ -67,6 +67,9 @@ class Settings:
     read_timeout: float = 300.0
     # Read timeout for the forwarded deepsearch-api request (it bundles, then answers).
     fallback_timeout: float = 300.0
+    # Fetch the files the package's BOM gives one sha256 once, written under every path that
+    # has them (dedupe.py): about half the reads of a simready-assets download. Every file ships.
+    dedupe: bool = True
 
     @classmethod
     def from_env(cls, env: Optional[Mapping[str, str]] = None) -> "Settings":

pipeline/package_download/sizes.py

@@ -75,6 +75,9 @@ class SizesSettings:
     relist_after: float = 6 * 3600.0
     # Outside .build/restore/, which the index restore empties at every start.
     cache_dir: Path = Path("/app/hf-space-pipeline/.build/package_sizes")
+    # The run's own sums (enumerate's listing, shipped by export): read once the index is readable, i.e. once the
+    # seed has downloaded the run. A run without one (before 2026-10-19) is listed from HF as before.
+    precomputed: Path = Path("/app/hf-space-pipeline/.build/restore/package_sizes.json")
 
     @classmethod
     def from_env(cls, env: Optional[Mapping[str, str]] = None) -> "SizesSettings":
@@ -189,6 +192,7 @@ class PackageSizes:
         self._walked = False
         self._sums: Dict[Group, Sums] = {}
         self._refused: Dict[Group, float] = {}  # group -> wall time it may be asked again
+        self._precomputed_loaded = False
         self._next_hf = 0.0
         self._task: Optional[asyncio.Task] = None
         self._search: Optional[httpx.AsyncClient] = None
@@ -284,6 +288,9 @@ class PackageSizes:
             logger.info("package sizes: index not readable yet (%s); next try in %.0f s", exc, s.retry_interval)
             return s.retry_interval
         self._keys, self._walked = keys, True
+        if not self._precomputed_loaded:
+            self._precomputed_loaded = True
+            self.load_precomputed()
         now = self._wall()
         self._refused = {g: t for g, t in self._refused.items() if t > now}
         counts = Counter(v[0] for v in keys.values() if v)
@@ -445,6 +452,30 @@ class PackageSizes:
             logger.info("package sizes: %d group(s) from %s", loaded, folder)
         return loaded
 
+    def load_precomputed(self) -> int:
+        """The run's own sums (pipeline/enumerate_assets.py's write_package_sizes): every group they cover needs no
+        HF listing. Anything unreadable leaves the listing to do it, group by group."""
+        path = Path(self.settings.precomputed)
+        try:
+            groups = json.loads(path.read_text())["groups"]
+        except FileNotFoundError:
+            return 0
+        except (OSError, ValueError, KeyError, TypeError) as exc:
+            logger.warning("package sizes: ignoring %s (%s)", path, exc)
+            return 0
+        loaded, now = 0, self._wall()
+        for doc in groups if isinstance(groups, list) else []:
+            try:
+                group = Group(str(doc["kind"]), str(doc["repo_id"]), str(doc["revision"]))
+                self._sums[group] = Sums(
+                    int(doc["whole"]), {str(k): int(v) for k, v in doc["folders"].items()}, int(doc["files"]), now,
+                )
+                loaded += 1
+            except (ValueError, KeyError, TypeError, AttributeError) as exc:
+                logger.warning("package sizes: ignoring a group in %s (%s)", path.name, exc)
+        logger.info("package sizes: %d group(s) precomputed by the index run (%s)", loaded, path)
+        return loaded
+
     def _write_cache(self, group: Group, sums: Sums) -> None:
         folder = self._groups_dir()
         try:

pipeline/profiles.py

@@ -0,0 +1,290 @@
+"""Stage `profiles` — each asset's SimReady validation profile, from its package's validation sidecar.
+
+The Space's Profiles filter lists the values of `simready_metadata_validation__profile`. USD Search indexes that
+property only from the root USD's `SimReady_Metadata.validation` block, which NVIDIA's and LG's packages author.
+Packages made with `simready-manage submit` record the profile in `<package>/.metadata/<package>_validation.json`
+instead (validation never edits the USD), so Robotiq's grippers (Robot-Gripper) and Standard Bots' arms (Robot-Body)
+reached the index with no profile at all. Between `categories` and `export`, this stage reads that sidecar at the
+package's pinned revision for every partner (non-NVIDIA) package and every asset still without a profile, and writes
+`simready_metadata_validation__profile` and `..._profile_version` from it:
+
+  * the sidecar wins where the USD says something else (it is the validation record); the stage lists every such
+    disagreement;
+  * a package without a sidecar (404) keeps whatever its USD says;
+  * NVIDIA packages whose USD carries a profile are not read (their packages have no sidecar).
+
+The sidecar is `{"<root USD, package-relative>": {"profile_id", "profile_version", ...}}` as `simready-manage`
+writes it; a top-level `profile` / `profile_version` (the shape simready-central's docs describe) is read too. A read
+that still fails after PROFILES_TRIES attempts fails the stage; nothing is written then. The record goes to
+.build/profiles.json.
+
+    python -m pipeline.run_pipeline profiles --env-file ... [--dry-run]
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from collections import Counter
+from dataclasses import dataclass
+from typing import Dict, Iterable, List, Optional, Tuple
+from urllib.parse import quote
+
+from . import hf_addressing
+from .categories import _scroll, _url_of, load_vendors, prop_values, with_property
+from .config import BUILD_DIR, Config
+
+PROPERTY = "simready_metadata_validation__profile"
+VERSION_PROPERTY = "simready_metadata_validation__profile_version"
+REPORT = BUILD_DIR / "profiles.json"
+
+
+@dataclass(frozen=True)
+class PackageRef:
+    host: str
+    repo_id: str  # "<owner>/<name>" of a dataset repo
+    revision: str
+    package: str  # the package folder at the repo root
+    path: str  # the asset's file, relative to the package folder
+
+    @property
+    def owner(self) -> str:
+        return self.repo_id.split("/")[0]
+
+    def sidecar_url(self) -> str:
+        rel = f"{self.package}/.metadata/{self.package}_validation.json"
+        return f"https://{self.host}/datasets/{self.repo_id}/resolve/{self.revision}/{quote(rel)}"
+
+
+# ── pure rules ───────────────────────────────────────────────────────────────
+
+
+def package_ref(url: str) -> Optional[PackageRef]:
+    """The dataset package an indexed asset belongs to, from its hf:// address or its public
+    https://<host>/datasets/<owner>/<name>/blob|resolve/<rev>/<path> URL. None for anything else."""
+    if url.startswith("hf://"):
+        try:
+            addr = hf_addressing.parse(url)
+        except hf_addressing.AddressError:
+            return None
+        if addr.repo_type != "dataset" or "/" not in addr.path:
+            return None
+        package, path = addr.path.split("/", 1)
+        return PackageRef(addr.host, addr.repo_id, addr.revision or "main", package, path)
+    parts = url.split("/")
+    # https: '' <host> datasets <owner> <name> blob|resolve <rev> <package> <path...>
+    if len(parts) > 9 and parts[0] in ("https:", "http:") and parts[3] == "datasets" and parts[6] in ("blob", "resolve"):
+        return PackageRef(parts[2], f"{parts[4]}/{parts[5]}", parts[7], parts[8], "/".join(parts[9:]))
+    return None
+
+
+def needs_sidecar(ref: Optional[PackageRef], props: Iterable[Dict], vendors: Dict) -> bool:
+    """Read the sidecar for a partner package, or for any asset whose USD gave no profile."""
+    if ref is None:
+        return False
+    return ref.owner.lower() not in vendors["own"] or not prop_values(props, PROPERTY)
+
+
+def _profile_of(entry: Dict) -> Optional[Tuple[str, str]]:
+    profile = str(entry.get("profile_id") or entry.get("profile") or "").strip()
+    if not profile:
+        return None
+    return profile, str(entry.get("profile_version") or "").strip()
+
+
+def profile_from_sidecar(doc, path: str) -> Tuple[Optional[Tuple[str, str]], str]:
+    """((profile, version), "ok") from a validation sidecar for the asset at package-relative `path`, or
+    (None, why): "malformed", "no-profile" or "ambiguous" (several USDs with different profiles, none of them `path`)."""
+    if not isinstance(doc, dict):
+        return None, "malformed"
+    top = _profile_of(doc)
+    if top:
+        return top, "ok"
+    entries = {k: _profile_of(v) for k, v in doc.items() if isinstance(v, dict)}
+    entries = {k: v for k, v in entries.items() if v}
+    if path in entries:
+        return entries[path], "ok"
+    found = set(entries.values())
+    if len(found) == 1:
+        return found.pop(), "ok"
+    return None, "ambiguous" if found else "no-profile"
+
+
+def with_profile(props: List[Dict], profile: str, version: str) -> List[Dict]:
+    """`props` carrying exactly this profile (and version, when the sidecar names one). A version the sidecar leaves
+    out is kept unless the profile itself changed. The same list back when nothing changes."""
+    current = prop_values(props, PROPERTY)
+    out = props
+    if current != [profile]:
+        out = with_property(out, PROPERTY, [profile])
+        if not version:
+            out = with_property(out, VERSION_PROPERTY, [])
+    if version and prop_values(out, VERSION_PROPERTY) != [version]:
+        out = with_property(out, VERSION_PROPERTY, [version])
+    return out
+
+
+# ── reading the sidecars ─────────────────────────────────────────────────────
+
+
+class SidecarError(RuntimeError):
+    pass
+
+
+def fetch_sidecar(http, url: str, tries: int, backoff_s: float) -> Tuple[str, object]:
+    """("ok", parsed sidecar), ("missing", None) when the package has none (404), or ("malformed", None) when it
+    is not JSON. Retries 429/5xx and transport errors; anything else, or the last failure, raises SidecarError."""
+    last = ""
+    for attempt in range(tries):
+        if attempt:
+            time.sleep(backoff_s * (2 ** (attempt - 1)))
+        try:
+            resp = http.get(url)
+        except Exception as e:  # noqa: BLE001 — httpx transport errors; retried, then reported
+            last = f"{type(e).__name__}: {e}"
+            continue
+        if resp.status_code == 404:
+            return "missing", None
+        if resp.status_code == 200:
+            try:
+                return "ok", resp.json()
+            except ValueError:
+                return "malformed", None
+        last = f"HTTP {resp.status_code}"
+        if resp.status_code != 429 and resp.status_code < 500:
+            break
+    raise SidecarError(f"{url}: {last}")
+
+
+def _headers() -> Dict[str, str]:
+    # Public datasets read anonymously; $HF_TOKEN (environment only, as everywhere in this pipeline) when set.
+    token = os.environ.get("HF_TOKEN")
+    return {"Authorization": f"Bearer {token}"} if token else {}
+
+
+# ── the stage ────────────────────────────────────────────────────────────────
+
+
+def run(cfg: Config, dry_run: bool = False) -> int:
+    import httpx
+
+    vendors = load_vendors()
+    base, index = cfg.opensearch_url, cfg.main_index
+    tries = max(1, int(os.environ.get("PROFILES_TRIES", "3")))
+    interval = float(os.environ.get("PROFILES_READ_INTERVAL_S", "0.25"))
+    outcomes: Counter = Counter()
+    values: Counter = Counter()
+    sources: Counter = Counter()
+    disagreements: List[Dict] = []
+    without: List[str] = []
+    errors: List[str] = []
+    updates: List[Tuple[str, List[Dict]]] = []
+    expected: Dict[str, Tuple[List[str], List[str]]] = {}
+    read = 0
+    with httpx.Client(timeout=60.0, follow_redirects=True) as index_http, \
+            httpx.Client(timeout=60.0, follow_redirects=True, headers=_headers()) as hf_http:
+        for hit in _scroll(index_http, base, index):
+            src = hit.get("_source") or {}
+            props = src.get("usd_properties")
+            if props is not None and not isinstance(props, list):
+                print(f"ERROR: {hit['_id']}: usd_properties is a {type(props).__name__}, not a list — not touching the index")
+                return 1
+            props = props or []
+            url = _url_of(src)
+            ref = package_ref(url)
+            usd = prop_values(props, PROPERTY)
+            new_props = props
+            if needs_sidecar(ref, props, vendors):
+                if read:
+                    time.sleep(interval)
+                read += 1
+                try:
+                    state, doc = fetch_sidecar(hf_http, ref.sidecar_url(), tries, backoff_s=2.0)
+                except SidecarError as e:
+                    errors.append(str(e))
+                    continue
+                if state == "missing":
+                    outcomes["no-sidecar"] += 1
+                else:
+                    found, why = profile_from_sidecar(doc, ref.path)
+                    outcomes[why] += 1
+                    if found:
+                        profile, version = found
+                        if usd and usd != [profile]:
+                            disagreements.append({"url": url, "usd": usd, "sidecar": profile})
+                        new_props = with_profile(props, profile, version)
+            else:
+                outcomes["usd-only"] += 1
+            final = prop_values(new_props, PROPERTY)
+            values.update(final)
+            sources["sidecar" if new_props is not props else ("usd" if final else "none")] += 1
+            if not final:
+                without.append(url)
+            if new_props is not props:
+                updates.append((hit["_id"], new_props))
+                expected[hit["_id"]] = (final, prop_values(new_props, VERSION_PROPERTY))
+
+        print(f"{sum(sources.values())} assets in {index}; {read} sidecar(s) read; outcomes: {dict(outcomes)}; "
+              f"{len(updates)} doc(s) to update")
+        print(f"{PROPERTY}: {json.dumps(dict(values), sort_keys=True)}")
+        for d in disagreements:
+            print(f"sidecar overrides the USD: {d['url']}: {d['usd']} -> {d['sidecar']}")
+        if without:
+            print(f"left without {PROPERTY} ({len(without)}): {without[:20]}")
+        REPORT.parent.mkdir(parents=True, exist_ok=True)
+        REPORT.write_text(json.dumps({
+            "property": PROPERTY, "index": index, "assets": sum(sources.values()), "sidecars_read": read,
+            "outcomes": dict(outcomes), "values": dict(sorted(values.items())), "disagreements": disagreements,
+            "without": without, "errors": errors, "updated": len(updates), "dry_run": dry_run,
+        }, indent=1) + "\n")
+        if errors:
+            print("ERROR: sidecar read failed:\nERROR: " + "\nERROR: ".join(errors[:20]))
+            return 1
+        if dry_run:
+            print("dry run: nothing written")
+            return 0
+        for i in range(0, len(updates), 200):
+            lines = []
+            for doc_id, props in updates[i:i + 200]:
+                lines.append(json.dumps({"update": {"_index": index, "_id": doc_id}}))
+                lines.append(json.dumps({"doc": {"usd_properties": props}}))
+            resp = index_http.post(f"{base}/_bulk", content="\n".join(lines) + "\n",
+                                   headers={"content-type": "application/x-ndjson"})
+            resp.raise_for_status()
+            if resp.json().get("errors"):
+                bad = [it for it in resp.json()["items"] if it.get("update", {}).get("error")][:3]
+                print(f"ERROR: bulk update failed: {json.dumps(bad)[:800]}")
+                return 1
+        index_http.post(f"{base}/{index}/_refresh").raise_for_status()
+        # Read back: the whole index holds the counted profiles, and every updated doc exactly its own.
+        written: Counter = Counter()
+        wrong: List[str] = []
+        for hit in _scroll(index_http, base, index):
+            props = (hit.get("_source") or {}).get("usd_properties") or []
+            written.update(prop_values(props, PROPERTY))
+            want = expected.get(hit["_id"])
+            if want and want != (prop_values(props, PROPERTY), prop_values(props, VERSION_PROPERTY)):
+                wrong.append(hit["_id"])
+        if written != values or wrong:
+            print(f"ERROR: read-back differs: wrote {dict(values)} but the index holds {dict(written)}; docs: {wrong[:10]}")
+            return 1
+    print(f"wrote {PROPERTY} on {len(updates)} doc(s); the index holds {json.dumps(dict(written), sort_keys=True)}")
+    return 0
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    import argparse
+    from pathlib import Path
+
+    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    ap.add_argument("--env-file", type=Path, default=None)
+    ap.add_argument("--dry-run", action="store_true")
+    a = ap.parse_args(argv)
+    return run(Config.load(a.env_file), dry_run=a.dry_run)
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(main())

pipeline/restore.py

@@ -25,6 +25,7 @@ from __future__ import annotations
 
 import gzip
 import json
+import os
 import shutil
 import subprocess
 from pathlib import Path
@@ -77,6 +78,21 @@ OPENSEARCH_TIMEOUT = 120.0
 BULK_TIMEOUT = 300.0
 
 
+def keep_derived_source_off(body: dict, mappings_json: str = "") -> None:
+    """RESTORE_KNN_DERIVED_SOURCE=off (the restore stage sets it; the Space's seed does not): create a k-NN index with
+    `index.knn.derived_source.enabled: false`, as the workers' own indexes are. build_create_body strips that
+    setting, so a restored index falls back to derived source ON, and OpenSearch then rebuilds the nested
+    `siglip2-embedding` vectors on read with one vector per segment: the next export writes that one vector onto
+    every document of the segment (run 20261012-9d9f42a: 1,399 of 1,404). The Space only searches its index (the
+    k-NN graph is built from the loaded vectors), so it keeps the default. Set it to `on` to restore the old behaviour."""
+    if os.environ.get("RESTORE_KNN_DERIVED_SOURCE", "").strip().lower() not in ("off", "false", "0"):
+        return
+    index_settings = body.setdefault("settings", {}).setdefault("index", {})
+    if str(index_settings.get("knn", "")).lower() == "true" or '"knn_vector"' in mappings_json:
+        index_settings["knn.derived_source.enabled"] = "false"
+        print("    k-NN derived source off for this index (RESTORE_KNN_DERIVED_SOURCE=off)")
+
+
 def download(cfg: Config, prefix: str, dest: Path = RESTORE_DIR) -> None:
     # dest is overridable so two independent restores (e.g. the HF Space's
     # OpenSearch and Neo4j restores, run by separate supervisord programs —
@@ -159,6 +175,7 @@ def restore_opensearch(cfg: Config, manifest: Dict) -> None:
                 os_dir / f"{index}.mapping.json.gz",
                 number_of_shards=cfg.opensearch_number_of_shards,
             )
+            keep_derived_source_off(body, json.dumps(body.get("mappings", {})))
             response = httpx.put(f"{cfg.opensearch_url}/{index}", json=body, timeout=OPENSEARCH_TIMEOUT)
             if response.status_code >= 400:
                 # raise_for_status()'s own message is just "403 Forbidden" —
@@ -236,6 +253,8 @@ def run(cfg: Config, fresh: bool = False) -> int:
     manifest = json.loads((RESTORE_DIR / "manifest.json").read_text())
 
     print("\nRestoring OpenSearch ...")
+    # This restore feeds an index that `export` dumps again: keep k-NN derived source off (keep_derived_source_off).
+    os.environ.setdefault("RESTORE_KNN_DERIVED_SOURCE", "off")
     restore_opensearch(cfg, manifest)
 
     print("\nRestoring Neo4j ...")

pipeline/run_pipeline.py

@@ -6,7 +6,8 @@ Stages run in order and each is independently re-runnable:
 
     enumerate  corpus.json -> exact asset list (N) + manifest
     index      bring the stack up, restore the baseline, clear, submit the
-               delta, wait
+               delta, wait, then the image-overrides, categories and
+               profiles steps
     export     elasticdump the indexes; neo4j-admin dump the graph
     upload     publish the artifact set to S3
     bake       build images with the data pre-loaded
@@ -20,7 +21,10 @@ Stages run in order and each is independently re-runnable:
     all        every stage in order. `hierarchy` runs right after `verify`
                with --new-only (only new Q-codes are placed; nothing placed
                moves), on the verified stack; a failure there only warns —
-               the index still ships, with the previous run's tree.
+               the index still ships, with the previous run's tree — but the
+               stage is required before deploying: deploy-space refuses a run
+               with no published hierarchy. After `--stop verify`, run it with
+               --run-id <run>.
 
     deploy-space  deploy the hf_space image to a Hugging Face Space (not
                   part of `all` — run manually after `push`; see
@@ -76,9 +80,14 @@ def stage_up(cfg: Config, args) -> int:
     if not (BUILD_DIR / "settings.yaml").exists() or not (BUILD_DIR / "plugin.config.yaml").exists():
         print("`.build/settings.yaml`/`.build/plugin.config.yaml` are missing — run `enumerate` first.")
         return 1
-    if cfg.siglip2_nvcf:
+    if cfg.indexing_siglip2_nvcf:
         print(f"siglip2 backend: nvcf ({cfg.get('SIGLIP2_NVCF_ENDPOINT', 'grpc.nvcf.nvidia.com:443')}) "
               f"— no local GPU needed for embeddings.")
+    else:
+        print(f"siglip2 backend: local ({cfg.get('SIGLIP2_IMAGE', 'usdsearch-siglip2-triton')}, needs a GPU)")
+    if cfg.storage_api_backend and not cfg.hf_token():
+        print("WARNING: no Hugging Face token ($HF_TOKEN or `hf auth login`) — hf-storage-api runs "
+              "anonymously and HF rate-limits it (429s) on anything but a small corpus.")
     if cfg.vlm_metadata_enabled:
         cfg.require_vlm_metadata_credentials()
         model = os.environ.get("USDSEARCH_VISION_METADATA_MODEL", "").strip() or "azure/openai/gpt-5.6-sol (default)"
@@ -133,20 +142,66 @@ def stage_wait(cfg: Config, args) -> int:
     return run(cfg)
 
 
+# The steps `index` runs after `wait`. Each writes into the index, so `export` refuses to run until all three have
+# succeeded after the latest `wait`: resuming with `wait` and then `all --start export` used to skip them silently
+# (2026-10-16), shipping an index without image overrides, the OEM category tree or partner profiles.
+ENRICHMENT_STEPS = ("image-overrides", "categories", "profiles")
+STAGE_MARKERS = BUILD_DIR / "stages"
+
+
+def _mark_done(step: str, args) -> None:
+    if not getattr(args, "dry_run", False):
+        STAGE_MARKERS.mkdir(parents=True, exist_ok=True)
+        (STAGE_MARKERS / f"{step}.done").touch()
+
+
+def enrichment_missing() -> List[str]:
+    """The ENRICHMENT_STEPS that haven't succeeded since the latest `wait` (its .build/index-report.json)."""
+    report = BUILD_DIR / "index-report.json"
+    waited = report.stat().st_mtime if report.exists() else 0.0
+    return [
+        step for step in ENRICHMENT_STEPS
+        if not (STAGE_MARKERS / f"{step}.done").exists()
+        or (STAGE_MARKERS / f"{step}.done").stat().st_mtime < waited
+    ]
+
+
+def stage_image_overrides(cfg: Config, args) -> int:
+    from .image_overrides import run
+    code = run(cfg, dry_run=args.dry_run)
+    if code == 0:
+        _mark_done("image-overrides", args)
+    return code
+
+
 def stage_categories(cfg: Config, args) -> int:
     from .categories import run
-    return run(cfg, dry_run=args.dry_run)
+    code = run(cfg, dry_run=args.dry_run)
+    if code == 0:
+        _mark_done("categories", args)
+    return code
+
+
+def stage_profiles(cfg: Config, args) -> int:
+    from .profiles import run
+    code = run(cfg, dry_run=args.dry_run)
+    if code == 0:
+        _mark_done("profiles", args)
+    return code
 
 
 def stage_index(cfg: Config, args) -> int:
-    """Bring the stack up, restore the baseline, clear, submit the delta, wait, write central:category."""
+    """Bring the stack up, restore the baseline, clear, submit the delta, wait, write central:category and the
+    validation profiles the packages' sidecars name."""
     steps = (
         ("up", stage_up),
         ("restore", stage_restore),
         ("clear", stage_clear),
         ("submit", stage_submit),
         ("wait", stage_wait),
+        ("image-overrides", stage_image_overrides),
         ("categories", stage_categories),
+        ("profiles", stage_profiles),
     )
     for step, fn in steps:
         print(f"\n{'=' * 70}\n== index: {step}\n{'=' * 70}")
@@ -158,6 +213,14 @@ def stage_index(cfg: Config, args) -> int:
 
 
 def stage_export(cfg: Config, args) -> int:
+    missing = enrichment_missing()
+    if missing:
+        print(
+            f"refusing to export: {', '.join(missing)} not run since the last `wait`. They write into the index "
+            f"(image overrides, central:category, partner profiles). Run them in order, then export:\n"
+            + "".join(f"    python -m pipeline.run_pipeline {step} --env-file ...\n" for step in missing)
+        )
+        return 1
     from .export import run
     return run(cfg)
 
@@ -247,6 +310,10 @@ def stage_all(cfg: Config, args) -> int:
             args.keep_up = keep_up or hierarchy_next  # the hierarchy stage reads the verified stack
         if name == "hierarchy":
             args.new_only = True
+            # This run, not latest.json's: under --no-latest that is still the previous run, whose tree the Spaces
+            # serve (2026-10-19: 20261019-9f72902's hierarchy started publishing into 20261015-09c47a6/hierarchy/).
+            from .enumerate_assets import load_manifest
+            args.run_id = getattr(args, "run_id", None) or load_manifest()["run_id"]
         try:
             code = fn(cfg, args)
         except (SystemExit, Exception) as e:  # noqa: BLE001 — only `hierarchy` may fail softly
@@ -305,7 +372,9 @@ STAGES = {
     "submit": stage_submit,
     "wait": stage_wait,
     "status": stage_wait,
+    "image-overrides": stage_image_overrides,
     "categories": stage_categories,
+    "profiles": stage_profiles,
     "index": stage_index,
     "export": stage_export,
     "upload": stage_upload,

pipeline/status.py

@@ -119,6 +119,12 @@ def classify(doc: Dict, plugins: Tuple[str, ...]) -> Dict[str, Tuple[str, str]]:
 async def poll_until_done(cfg: Config, urls: List[str]) -> Tuple[Dict[str, Tally], List[str]]:
     """Poll the outstanding set until every (asset, plugin) pair is terminal."""
     interval = cfg.get_float("STATUS_POLL_INTERVAL", 15.0)
+    # Each asset-status request makes info-endpoint list the asset through the
+    # storage backend. With storage_api that is an HF tree listing, so polling
+    # ~2,600 assets every cycle exhausted HF's API quota (429s) and starved the
+    # workers. While jobs are still queued, watch only the Redis queues and do
+    # a full per-asset poll at most every STATUS_FULL_POLL_INTERVAL seconds.
+    full_poll_interval = cfg.get_float("STATUS_FULL_POLL_INTERVAL", 600.0)
     timeout = cfg.get_float("STATUS_TIMEOUT", 21600.0)
     concurrency = cfg.get_int("STATUS_CONCURRENCY", 16)
     redis_url = f"redis://localhost:{cfg.get_int('REDIS_HOST_PORT', 6380)}"
@@ -132,6 +138,7 @@ async def poll_until_done(cfg: Config, urls: List[str]) -> Tuple[Dict[str, Tally
     redis_client = connect(redis_url)
 
     limits = httpx.Limits(max_connections=concurrency * 2)
+    last_full_poll: Optional[float] = None
     async with httpx.AsyncClient(limits=limits, timeout=httpx.Timeout(30.0)) as client:
         while outstanding:
             if time.monotonic() > deadline:
@@ -139,6 +146,15 @@ async def poll_until_done(cfg: Config, urls: List[str]) -> Tuple[Dict[str, Tally
                 dump_worker_logs(cfg)
                 break
 
+            if last_full_poll is not None and time.monotonic() - last_full_poll < full_poll_interval:
+                depths = await queue_depths(redis_client, plugins)
+                queued = sum(v for k, v in depths.items() if k != "results")
+                if queued or depths["results"]:
+                    print(f"  queued={queued} results={depths['results']}", flush=True)
+                    await asyncio.sleep(interval)
+                    continue
+            last_full_poll = time.monotonic()
+
             async def check(url: str) -> Tuple[str, Optional[Dict]]:
                 async with semaphore:
                     return url, await fetch_status(client, cfg.info_endpoint, url)

services/explorer/dist/build-info.json

@@ -1,7 +1,7 @@
 {
   "product": "simready-central-slim-explorer",
-  "version": "slim-ui-8692f52b",
-  "bundle_id": "main.94a34e97.js",
-  "built_at": "2026-10-19T09:20:23Z",
+  "version": "slim-ui-e4522a3f",
+  "bundle_id": "main.01872f5b.js",
+  "built_at": "2026-10-18T17:30:52Z",
   "flags": {"public_mode": true, "email_auth": false, "create_ui": false, "hierarchy_editor": false, "create_search": false, "auth_ui": false, "credentials_ui": false, "index_management": false, "inference_backend": "hub", "vlm_asset_refs": false, "validation_image_max_px": 448, "validation_max_retries": 0, "validation_max_concurrent": 32, "validation_tail_concurrent": 8, "validation_eager_count": 50, "validation_batch_stream": true, "library_idle_prefetch": false, "validation_score_scale": "percent", "parse_gate_fields": true, "parse_fast_path": true, "featured_landing": true, "hf_download_commands": true, "dependency_views": true, "model_viewer": false, "library_viewer": true, "library_media": "runtime", "spell_correction": true, "downloads": "runtime", "submit_content": "runtime", "third_party_notices": "runtime"}
 }

services/explorer/dist/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/ds.png"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="SimReady Central — SimReady asset search"/><meta name="ui-version" content="slim-ui-8692f52b"><script src="/config.js"></script><title>SimReady Central</title><script defer="defer" src="/static/js/main.94a34e97.js"></script><link href="/static/css/main.ed151f6a.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/ds.png"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="SimReady Central — SimReady asset search"/><meta name="ui-version" content="slim-ui-e4522a3f"><script src="/config.js"></script><title>SimReady Central</title><script defer="defer" src="/static/js/main.01872f5b.js"></script><link href="/static/css/main.ed151f6a.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

services/explorer/dist/third-party-notices.txt

@@ -1,4 +1,4 @@
-Third-party notices — SimReady Central Explorer slim-ui-8692f52b
+Third-party notices — SimReady Central Explorer slim-ui-e4522a3f
 
 The software in this web application's browser bundle includes the packages below, each under its own
 license. Each entry gives the package, its version, its license and the license / notice text it ships.