Deploy simready-assets run 20261020-a3cb7f2-captions (GPU variant) (part 2)

pipeline/config.py

@@ -419,6 +419,11 @@ class Config:
             # a text query's judge reads each hit's path (below its dataset or Isaac content root) and vision
             # caption, and an image + words query asks whether the candidate matches both (usdsearch !279).
             "USDSEARCH_VISION_VALIDATION_IMAGE_MAX_PX": "448",
+            # Views and query images go to the engine as JPEG q90 (transparency on white), not PNG: less API CPU per
+            # check. Needs wheels with usdsearch !303 (older ones ignore it). Part of the verdict cache identity and of
+            # check_image_cache.py's folder (it keys on the whole validation config), so the warm-up prepares JPEGs.
+            # Off: delete the Space Variable and restart; a deploy sets it again while this line is here.
+            "USDSEARCH_VISION_VALIDATION_IMAGE_FORMAT": "jpeg",
             # Each stored view is fitted and encoded for its check once per container, not once per check (most
             # of a check's API CPU): docker/hf-space/check_image_cache.py, GPU image only (the CPU image has no
             # hook: the variable does nothing there). Off: delete the Space Variable and restart; a deploy sets it

pipeline/export.py

@@ -275,8 +275,35 @@ def export_neo4j(cfg: Config) -> Dict:
     }
 
 
+def package_sizes_problem(manifest: Dict, path: Path) -> str:
+    """Why `path` is not this run's package_sizes.json ("" when it is): its (repo, revision) groups must be exactly
+    the manifest's hf_repos. On 2026-10-20 a test overwrote .build/package_sizes.json with its fixture
+    (nvidia/repo-a@main) and export shipped it with run 20261020-a3cb7f2-captions; this is the check that run
+    lacked. Only a storage_api run has one."""
+    if manifest.get("storage_backend") != "storage_api":
+        return ""
+    if not path.exists():
+        return f"{path.name} is missing: re-run `enumerate`"
+    try:
+        groups = json.loads(path.read_text()).get("groups") or []
+    except (OSError, ValueError, AttributeError) as e:
+        return f"{path.name} is unreadable ({e}): re-run `enumerate`"
+    have = {(g.get("repo_id"), g.get("revision")) for g in groups if isinstance(g, dict)}
+    want = {(repo_id, revision) for _host, _type, repo_id, revision in manifest.get("hf_repos") or []}
+    if have != want:
+        extra, missing = sorted(have - want), sorted(want - have)
+        return (f"{path.name} is not this run's: {len(extra)} (repo, revision) group(s) the run doesn't list "
+                f"{extra[:3]}, {len(missing)} of its own missing {missing[:3]}. Re-run `enumerate` with this "
+                f"run's RUN_ID, or restore the file from the run's S3 prefix.")
+    return ""
+
+
 def run(cfg: Config) -> int:
     manifest = load_manifest()
+    problem = package_sizes_problem(manifest, BUILD_DIR / "package_sizes.json")
+    if problem:
+        print(f"export: {problem}")
+        return 1
     ARTIFACTS.mkdir(parents=True, exist_ok=True)
 
     print("Exporting OpenSearch ...")

pipeline/profiles.py

@@ -18,6 +18,12 @@ writes it; a top-level `profile` / `profile_version` (the shape simready-central
 that still fails after PROFILES_TRIES attempts fails the stage; nothing is written then. The record goes to
 .build/profiles.json.
 
+A sidecar at a pinned revision (a 40-hex commit) never changes, so every one read there is kept in
+.build/profiles-sidecars.json (the 404s too) and later runs read only packages pinned since: a run that re-pins
+nothing reads none, where reading all ~1,060 partner sidecars at PROFILES_READ_INTERVAL_S took ~9 minutes. A sidecar
+at a branch (`main`) is read every time. Within a run each sidecar is read once, however many of its package's
+assets the index holds. PROFILES_CACHE=off reads everything again (and still refreshes the file).
+
     python -m pipeline.run_pipeline profiles --env-file ... [--dry-run]
 """
 
@@ -25,9 +31,11 @@ from __future__ import annotations
 
 import json
 import os
+import re
 import time
 from collections import Counter
 from dataclasses import dataclass
+from pathlib import Path
 from typing import Dict, Iterable, List, Optional, Tuple
 from urllib.parse import quote
 
@@ -38,6 +46,8 @@ from .config import BUILD_DIR, Config
 PROPERTY = "simready_metadata_validation__profile"
 VERSION_PROPERTY = "simready_metadata_validation__profile_version"
 REPORT = BUILD_DIR / "profiles.json"
+CACHE = BUILD_DIR / "profiles-sidecars.json"
+PINNED_REVISION = re.compile(r"^[0-9a-f]{40}$")
 
 
 @dataclass(frozen=True)
@@ -52,6 +62,10 @@ class PackageRef:
     def owner(self) -> str:
         return self.repo_id.split("/")[0]
 
+    @property
+    def pinned(self) -> bool:
+        return bool(PINNED_REVISION.match(self.revision))
+
     def sidecar_url(self) -> str:
         rel = f"{self.package}/.metadata/{self.package}_validation.json"
         return f"https://{self.host}/datasets/{self.repo_id}/resolve/{self.revision}/{quote(rel)}"
@@ -157,6 +171,24 @@ def fetch_sidecar(http, url: str, tries: int, backoff_s: float) -> Tuple[str, ob
     raise SidecarError(f"{url}: {last}")
 
 
+def load_cache(path: Path) -> Dict[str, Tuple[str, object]]:
+    """sidecar URL -> (state, parsed sidecar) as fetch_sidecar returned it; {} when there is no usable file."""
+    try:
+        raw = json.loads(path.read_text())
+    except (OSError, ValueError):
+        return {}
+    entries = raw.get("sidecars") if isinstance(raw, dict) else None
+    return {url: (v[0], v[1]) for url, v in (entries or {}).items()
+            if isinstance(v, list) and len(v) == 2 and v[0] in ("ok", "missing", "malformed")}
+
+
+def save_cache(path: Path, cache: Dict[str, Tuple[str, object]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text(json.dumps({"sidecars": {url: list(v) for url, v in sorted(cache.items())}}) + "\n")
+    tmp.replace(path)
+
+
 def _headers() -> Dict[str, str]:
     # Public datasets read anonymously; $HF_TOKEN (environment only, as everywhere in this pipeline) when set.
     token = os.environ.get("HF_TOKEN")
@@ -182,6 +214,11 @@ def run(cfg: Config, dry_run: bool = False) -> int:
     updates: List[Tuple[str, List[Dict]]] = []
     expected: Dict[str, Tuple[List[str], List[str]]] = {}
     read = 0
+    reused = 0
+    use_cache = os.environ.get("PROFILES_CACHE", "on").strip().lower() not in ("off", "0", "false", "no")
+    cache = load_cache(CACHE) if use_cache else {}
+    pinned = len(cache)
+    seen: Dict[str, Tuple[str, object]] = {}  # this run's reads, branch revisions included
     with httpx.Client(timeout=60.0, follow_redirects=True) as index_http, \
             httpx.Client(timeout=60.0, follow_redirects=True, headers=_headers()) as hf_http:
         for hit in _scroll(index_http, base, index):
@@ -196,14 +233,23 @@ def run(cfg: Config, dry_run: bool = False) -> int:
             usd = prop_values(props, PROPERTY)
             new_props = props
             if needs_sidecar(ref, props, vendors):
-                if read:
-                    time.sleep(interval)
-                read += 1
-                try:
-                    state, doc = fetch_sidecar(hf_http, ref.sidecar_url(), tries, backoff_s=2.0)
-                except SidecarError as e:
-                    errors.append(str(e))
-                    continue
+                sidecar_url = ref.sidecar_url()
+                known = seen.get(sidecar_url) or cache.get(sidecar_url)
+                if known:
+                    reused += 1
+                    state, doc = known
+                else:
+                    if read:
+                        time.sleep(interval)
+                    read += 1
+                    try:
+                        state, doc = fetch_sidecar(hf_http, sidecar_url, tries, backoff_s=2.0)
+                    except SidecarError as e:
+                        errors.append(str(e))
+                        continue
+                    seen[sidecar_url] = (state, doc)
+                    if ref.pinned:
+                        cache[sidecar_url] = (state, doc)
                 if state == "missing":
                     outcomes["no-sidecar"] += 1
                 else:
@@ -225,8 +271,10 @@ def run(cfg: Config, dry_run: bool = False) -> int:
                 updates.append((hit["_id"], new_props))
                 expected[hit["_id"]] = (final, prop_values(new_props, VERSION_PROPERTY))
 
-        print(f"{sum(sources.values())} assets in {index}; {read} sidecar(s) read; outcomes: {dict(outcomes)}; "
-              f"{len(updates)} doc(s) to update")
+        if len(cache) > pinned:
+            save_cache(CACHE, cache)  # before any error below: a retry reads only what failed
+        print(f"{sum(sources.values())} assets in {index}; {read} sidecar(s) read, {reused} already known "
+              f"({CACHE.name}: {len(cache)} pinned); outcomes: {dict(outcomes)}; {len(updates)} doc(s) to update")
         print(f"{PROPERTY}: {json.dumps(dict(values), sort_keys=True)}")
         for d in disagreements:
             print(f"sidecar overrides the USD: {d['url']}: {d['usd']} -> {d['sidecar']}")
@@ -235,6 +283,7 @@ def run(cfg: Config, dry_run: bool = False) -> int:
         REPORT.parent.mkdir(parents=True, exist_ok=True)
         REPORT.write_text(json.dumps({
             "property": PROPERTY, "index": index, "assets": sum(sources.values()), "sidecars_read": read,
+            "sidecars_reused": reused,
             "outcomes": dict(outcomes), "values": dict(sorted(values.items())), "disagreements": disagreements,
             "without": without, "errors": errors, "updated": len(updates), "dry_run": dry_run,
         }, indent=1) + "\n")