Automatic spelling vocabulary: /spell_vocab.json from the index (indexing #98 c8f9e69) + UI slim-ui-940a240b

docker/hf-space/nginx.conf.template

@@ -170,6 +170,20 @@ server {
         return 200 '{"index":"usdsearch-dev","complete":false,"generated_at":null,"basis":"package sizes are not available yet","total":0,"sizes":{}}';
     }
 
+    # The corpus's own words for the Explorer's spelling suggestion: package-download keeps the table current
+    # (pipeline/package_download/spell_vocab.py). Not up yet, or failing: a complete:false 200 (the Explorer keeps its
+    # baked list). A 404 (switched off) passes through.
+    location = /spell_vocab.json {
+        if ($request_method = OPTIONS) { add_header Content-Length 0 always; add_header Content-Type "text/plain" always; return 204; }
+        proxy_pass http://127.0.0.1:8020;
+        proxy_intercept_errors on;
+        error_page 500 502 503 504 = @spell_vocab_pending;
+    }
+    location @spell_vocab_pending {
+        default_type application/json;
+        return 200 '{"index":"usdsearch-dev","complete":false,"generated_at":null,"basis":"the spelling vocabulary is not available yet","assets":0,"count":0,"words":[]}';
+    }
+
     location /images {
         if ($request_method = OPTIONS) { add_header Content-Length 0 always; add_header Content-Type "text/plain" always; return 204; }
         rewrite ^/images(.*)$ /v3/images$1 break;

pipeline/package_download/__main__.py

@@ -11,6 +11,7 @@ from .app import create_app
 from .catalog import Catalog
 from .settings import Settings
 from .sizes import PackageSizes, SizesSettings
+from .spell_vocab import SpellVocab, VocabSettings
 
 
 def main() -> int:
@@ -31,8 +32,11 @@ def main() -> int:
     # /package_sizes.json: every indexed asset's download size, kept current in the background (sizes.py).
     sizes_settings = SizesSettings.from_env()
     sizes = PackageSizes(sizes_settings) if sizes_settings.enabled else None
+    # /spell_vocab.json: the corpus's own words, which the Explorer's "Did you mean" never questions (spell_vocab.py).
+    vocab_settings = VocabSettings.from_env()
+    vocab = SpellVocab(vocab_settings) if vocab_settings.enabled else None
     uvicorn.run(
-        create_app(settings, catalog=catalog, sizes=sizes), host=settings.host, port=settings.port, log_config=None
+        create_app(settings, catalog=catalog, sizes=sizes, vocab=vocab), host=settings.host, port=settings.port, log_config=None
     )
     return 0
 

pipeline/package_download/app.py

@@ -26,6 +26,7 @@ from . import bundle, telemetry
 from .catalog import Catalog
 from .settings import Settings
 from .sizes import PackageSizes
+from .spell_vocab import SpellVocab
 
 logger = logging.getLogger("package_download")
 
@@ -184,6 +185,7 @@ def create_app(
     url_mapping: Optional[UrlMapping] = None,
     fallback_transport: Optional[httpx.AsyncBaseTransport] = None,
     sizes: Optional[PackageSizes] = None,
+    vocab: Optional[SpellVocab] = None,
 ) -> FastAPI:
     settings = settings or Settings.from_env()
     to_internal, to_public, to_public_url = url_mapping or _default_url_mapping()
@@ -220,11 +222,15 @@ def create_app(
         )
         if sizes is not None:
             sizes.start()
+        if vocab is not None:
+            vocab.start()
         try:
             yield
         finally:
             if sizes is not None:
                 await sizes.stop()
+            if vocab is not None:
+                await vocab.stop()
             await app.state.fallback.aclose()
             if owned_source:
                 await source.close()
@@ -264,6 +270,15 @@ def create_app(
                 headers["Content-Encoding"] = encoding
             return Response(body, media_type="application/json", headers=headers)
 
+    if vocab is not None:
+        @app.get("/spell_vocab.json")
+        async def spell_vocab(request: Request):
+            body, encoding = vocab.body(request.headers.get("accept-encoding", ""))
+            headers = {"Cache-Control": "no-cache", "Vary": "Accept-Encoding"}
+            if encoding:
+                headers["Content-Encoding"] = encoding
+            return Response(body, media_type="application/json", headers=headers)
+
     async def forward(request: Request) -> Response:
         """Hand the request to deepsearch-api's own /v3/download/asset, unchanged."""
         params = [(k, v) for k, v in request.query_params.multi_items() if k != "scope"]

pipeline/package_download/spell_vocab.py

@@ -0,0 +1,355 @@
+"""``GET /spell_vocab.json``: the corpus's own words, so the Explorer's "Did you mean" never questions them.
+
+The Explorer (SimReady Central's ``utils/spellSuggest.js``) offers a spelling from the English word list for a typed
+word the list lacks. A word this corpus uses (a vendor, a product code stem, a file name part, a category label) is
+not a typo: the Explorer treats every word here like its own hard-coded protected list (the 2026-10-17 scan, kept as
+the fallback). This table replaces hand-curating that list after every re-index.
+
+Same lifecycle as ``sizes.py``: a background task, at start and every ``interval`` seconds, pages this Space's own
+index (deepsearch-api's query-less search, by cursor, with each hit's vision metadata) and reads the property
+inventory (deepsearch-api's ``stats/usd_properties``) and the restored Category tree. Words come from every asset's
+file name, folder segments, dataset org and name, vision caption, object_type, tags and queries, the inventory's
+name / class / category / vendor / type values, and the Category tree's labels and vendor names. Each text is split
+at non-letters, camelCase and digits; words of ``min_len``+ letters are kept, lowercased, de-duplicated; words the
+English list already has are left out (the Explorer never questions those). Until the walk and the inventory have
+both been read the table says ``complete: false`` (the Explorer keeps its baked list).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import gzip
+import hashlib
+import json
+import logging
+import os
+import re
+import time
+from dataclasses import dataclass, fields
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Awaitable, Callable, Iterable, Mapping, Optional, Set, Tuple
+from urllib.parse import unquote
+
+import httpx
+
+logger = logging.getLogger("package_download.spell_vocab")
+
+PREFIX = "SPELL_VOCAB_"
+BASIS = (
+    "words of every indexed asset's file name, folders, dataset, vision caption / object_type / tags / queries, "
+    "the property inventory's name, class, category and vendor values and the Category tree's labels; "
+    "English-list words left out"
+)
+# The vision bags deepsearch-api returns on a hit's source (return_vision_generated_metadata).
+VISION_BAGS = (
+    "plugin_thumbnail_to_vision_metadata_metadata_vlm_generated",
+    "plugin_rendering_to_vision_metadata_metadata_vlm_generated",
+)
+VISION_FIELDS = frozenset({"caption", "object_type", "tags", "queries"})
+# Inventory keys whose values are words a user types for an asset (the 2026-10-17 scan's rule).
+STATS_KEY = re.compile(r"(name|class|categor|vendor|manufact|brand|maker|author|profile|type|label|wikidata)", re.I)
+QCODE = re.compile(r"^Q\d+$")
+# A hex run of 8+ with 2+ digits (a hash, a uuid part) is no word: its letter fragments would be noise.
+HEXRUN = re.compile(r"(?=(?:[a-f]*\d){2})[0-9a-f]{8,}", re.I)
+# camelCase / ACRONYMWord / digits: "LGElectronics" -> LG, Electronics; "mser1130b" -> mser, b.
+PARTS = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+")
+
+
+@dataclass(frozen=True)
+class VocabSettings:
+    enabled: bool = True
+    index: str = "usdsearch-dev"
+    search_url: str = "http://127.0.0.1:8000/v3/deepsearch/search"
+    stats_url: str = "http://127.0.0.1:8000/v2/deepsearch/search/stats/usd_properties"
+    page_size: int = 500
+    search_user: str = "simready-central-svc-user"
+    search_password: str = "simready-central-svc-pass"
+    # nginx's /wikidata_hierarchy.json (the Category tree), restored with the index run; absent is fine.
+    hierarchy_path: Path = Path("/app/hf-space-pipeline/.build/restore/hierarchy/category_hierarchy.json")
+    # The dist's English list: its words are left out of the table (absent: nothing left out).
+    english_path: Path = Path("/usr/share/nginx/explorer/spell/en-words.txt")
+    interval: float = 900.0
+    retry_interval: float = 30.0
+    # The inventory (a multi-MB aggregation) is read again only when the walk's assets changed, or after this long.
+    reread_after: float = 6 * 3600.0
+    timeout: float = 120.0
+    min_len: int = 3
+    # Longer inventory values are descriptions, not names.
+    max_value_len: int = 120
+
+    @classmethod
+    def from_env(cls, env: Optional[Mapping[str, str]] = None) -> "VocabSettings":
+        env = os.environ if env is None else env
+        values = {}
+        for field in fields(cls):
+            raw = env.get(PREFIX + field.name.upper())
+            if raw is None or raw == "":
+                continue
+            default = field.default
+            if isinstance(default, bool):
+                values[field.name] = raw.strip().lower() in ("1", "true", "yes", "on")
+            elif isinstance(default, int):
+                values[field.name] = int(raw)
+            elif isinstance(default, float):
+                values[field.name] = float(raw)
+            elif isinstance(default, Path):
+                values[field.name] = Path(raw)
+            else:
+                values[field.name] = raw
+        return cls(**values)
+
+
+def words_of(text: Any, min_len: int = 3) -> Iterable[str]:
+    """Lowercase words of ``min_len``+ letters: split at non-letters, camelCase and digits (a camelCase run kept whole
+    too); hash-like hex runs dropped."""
+    if not isinstance(text, str) or not text:
+        return
+    for run in re.findall(r"[A-Za-z]+", HEXRUN.sub(" ", text)):
+        pieces = PARTS.findall(run)
+        if len(pieces) > 1 and len(run) >= min_len:
+            yield run.lower()  # "IsaacSim" is typed whole too
+        for w in pieces:
+            if len(w) >= min_len:
+                yield w.lower()
+
+
+def key_texts(base_key: str) -> Iterable[str]:
+    """``…/datasets/<org>/<name>/blob/<rev>/<path>``: org, name, every folder and the file stem (the revision never)."""
+    m = re.search(r"/(?:datasets/|spaces/)?([^/]+)/([^/]+)/(?:blob|resolve)/[^/]+/(.+)$", base_key or "")
+    if not m:
+        return
+    yield m.group(1)
+    yield m.group(2)
+    segs = unquote(m.group(3).split("?", 1)[0]).split("/")
+    yield from segs[:-1]
+    yield re.sub(r"\.[A-Za-z0-9]+$", "", segs[-1])
+
+
+def hit_texts(hit: Mapping[str, Any]) -> Iterable[str]:
+    source = hit.get("source") or {}
+    yield from key_texts(source.get("base_key") or "")
+    name = source.get("name")
+    if isinstance(name, str):
+        yield re.sub(r"\.[A-Za-z0-9]+$", "", name)
+    for bag in VISION_BAGS:
+        for item in source.get(bag) or []:
+            if isinstance(item, dict) and item.get("name") in VISION_FIELDS:
+                for v in item.get("value_text") or []:
+                    yield v
+    vgm = source.get("vision_generated_metadata")
+    if isinstance(vgm, dict):
+        for k, v in vgm.items():
+            if any(f in k for f in VISION_FIELDS):
+                yield from (v if isinstance(v, list) else [v])
+
+
+def stats_texts(doc: Mapping[str, Any], max_len: int) -> Iterable[str]:
+    for kv in doc.get("kv_pairs") or []:
+        if not isinstance(kv, dict):
+            continue
+        val = kv.get("value")
+        if STATS_KEY.search(str(kv.get("key", ""))) and isinstance(val, str) and len(val) <= max_len:
+            if not QCODE.match(val):
+                yield val
+
+
+def hierarchy_texts(doc: Any) -> Iterable[str]:
+    """Every ``label`` in the Category tree, its vendor names and its folder pins."""
+    if isinstance(doc, dict):
+        for k, v in doc.items():
+            if k == "label" and isinstance(v, str):
+                yield v
+            elif k == "vendors" and isinstance(v, dict):
+                for a, b in v.items():
+                    yield a
+                    if isinstance(b, str):
+                        yield b
+            else:
+                yield from hierarchy_texts(v)
+    elif isinstance(doc, list):
+        for v in doc:
+            yield from hierarchy_texts(v)
+
+
+class IndexUnavailable(Exception):
+    pass
+
+
+class SpellVocab:
+    """The word table and the task that keeps it current. ``body()`` is what the route sends."""
+
+    def __init__(
+        self,
+        settings: Optional[VocabSettings] = None,
+        *,
+        transport: Optional[httpx.AsyncBaseTransport] = None,
+        wall: Callable[[], float] = time.time,
+        sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+    ):
+        self.settings = settings or VocabSettings.from_env()
+        self._transport, self._wall, self._sleep = transport, wall, sleep
+        self._client: Optional[httpx.AsyncClient] = None
+        self._task: Optional[asyncio.Task] = None
+        self._english: Optional[Set[str]] = None
+        self._fingerprint = ""
+        self._stats_words: Optional[Set[str]] = None
+        self._stats_at = 0.0
+        self.words: list = []
+        self.assets = 0
+        self.complete = False
+        self._body = self._gz = b""
+        self._rebuild(None)
+
+    # --- serving ---------------------------------------------------------------------------------------------
+
+    def body(self, accept_encoding: str = "") -> Tuple[bytes, Optional[str]]:
+        if "gzip" in (accept_encoding or "").lower():
+            return self._gz, "gzip"
+        return self._body, None
+
+    def _rebuild(self, generated_at: Optional[float]) -> None:
+        doc = {
+            "index": self.settings.index,
+            "complete": self.complete,
+            "generated_at": (
+                datetime.fromtimestamp(generated_at, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+                if generated_at is not None else None
+            ),
+            "basis": BASIS,
+            "assets": self.assets,
+            "count": len(self.words),
+            "words": self.words,
+        }
+        self._body = json.dumps(doc, separators=(",", ":")).encode()
+        self._gz = gzip.compress(self._body, compresslevel=9, mtime=0)
+
+    # --- the task --------------------------------------------------------------------------------------------
+
+    def start(self) -> None:
+        self._task = asyncio.get_running_loop().create_task(self.run(), name="spell-vocab")
+
+    async def stop(self) -> None:
+        if self._task is not None:
+            self._task.cancel()
+            await asyncio.gather(self._task, return_exceptions=True)
+            self._task = None
+        if self._client is not None:
+            await self._client.aclose()
+            self._client = None
+
+    async def run(self) -> None:
+        while True:
+            try:
+                delay = await self.refresh()
+            except asyncio.CancelledError:
+                raise
+            except Exception:  # noqa: BLE001 — the table stays as it was; try again later
+                logger.exception("spell vocab: pass failed")
+                delay = self.settings.retry_interval
+            await self._sleep(delay)
+
+    def _http(self) -> httpx.AsyncClient:
+        s = self.settings
+        if self._client is None:
+            auth = (s.search_user, s.search_password) if s.search_user else None
+            self._client = httpx.AsyncClient(timeout=s.timeout, transport=self._transport, trust_env=False, auth=auth)
+        return self._client
+
+    def _english_words(self) -> Set[str]:
+        if self._english is None:
+            words: Set[str] = set()
+            try:
+                for line in Path(self.settings.english_path).read_text(encoding="utf-8").splitlines():
+                    w = line.split("\t", 1)[0].strip().lower()
+                    if w.isalpha():
+                        words.add(w)
+            except OSError as exc:
+                logger.info("spell vocab: no English list at %s (%s); every word kept", self.settings.english_path, exc)
+            self._english = words
+        return self._english
+
+    async def refresh(self) -> float:
+        """One pass: walk the index, read the inventory when needed, rebuild. Returns the delay to the next."""
+        s = self.settings
+        try:
+            hits = await self.walk_index()
+        except IndexUnavailable as exc:
+            logger.info("spell vocab: index not readable yet (%s); next try in %.0f s", exc, s.retry_interval)
+            return s.retry_interval
+        fingerprint = hashlib.sha256(
+            "\n".join(sorted(f"{(h.get('source') or {}).get('base_key')}|{(h.get('source') or {}).get('hash_value')}"
+                             for h in hits)).encode()
+        ).hexdigest()
+        now = self._wall()
+        if (self._stats_words is None or fingerprint != self._fingerprint
+                or now - self._stats_at >= s.reread_after):
+            try:
+                stats = await self._get_json(s.stats_url)
+            except IndexUnavailable as exc:
+                logger.info("spell vocab: inventory not readable (%s); next try in %.0f s", exc, s.retry_interval)
+                return s.retry_interval
+            self._stats_words = {w for t in stats_texts(stats, s.max_value_len) for w in words_of(t, s.min_len)}
+            self._stats_at = now
+        elif fingerprint == self._fingerprint and self.complete:
+            return s.interval  # nothing moved
+        words: Set[str] = set(self._stats_words)
+        for hit in hits:
+            for text in hit_texts(hit):
+                words.update(words_of(text, s.min_len))
+        words.update(w for t in hierarchy_texts(self._read_hierarchy()) for w in words_of(t, s.min_len))
+        english = self._english_words()
+        self.words = sorted(w for w in words if w not in english)
+        self.assets, self.complete, self._fingerprint = len(hits), True, fingerprint
+        self._rebuild(now)
+        logger.info(
+            "spell vocab: %d asset(s), %d word(s) (%d left out as English), %d bytes, %d gzipped (complete)",
+            len(hits), len(self.words), len(words) - len(self.words), len(self._body), len(self._gz),
+        )
+        return s.interval
+
+    def _read_hierarchy(self) -> Any:
+        path = Path(self.settings.hierarchy_path)
+        try:
+            return json.loads(path.read_text(encoding="utf-8"))
+        except FileNotFoundError:
+            return None
+        except (OSError, ValueError) as exc:
+            logger.warning("spell vocab: ignoring the Category tree at %s (%s)", path, exc)
+            return None
+
+    async def _get_json(self, url: str) -> Any:
+        try:
+            r = await self._http().get(url)
+            r.raise_for_status()
+            return r.json()
+        except (httpx.HTTPError, ValueError) as exc:
+            raise IndexUnavailable(f"{type(exc).__name__}: {exc}") from exc
+
+    async def walk_index(self) -> list:
+        """Every hit in the index with its vision metadata (deepsearch-api's query-less search, by cursor)."""
+        s = self.settings
+        hits: dict = {}
+        cursor = None
+        for _ in range(10_000):
+            body = {"limit": s.page_size, "return_metadata": True, "return_vision_generated_metadata": True,
+                    "return_usd_properties": False, "return_images": False}
+            if cursor:
+                body["cursor"] = cursor
+            try:
+                r = await self._http().post(s.search_url, json=body)
+                r.raise_for_status()
+                doc = r.json()
+            except (httpx.HTTPError, ValueError) as exc:
+                raise IndexUnavailable(f"{type(exc).__name__}: {exc}") from exc
+            page = doc.get("hits") or []
+            for hit in page:
+                key = (hit.get("source") or {}).get("base_key")
+                if isinstance(key, str) and key:
+                    hits[key] = hit
+            if not doc.get("has_more"):
+                return list(hits.values())
+            cursor = doc.get("next_cursor")
+            if not page or not cursor:  # never a complete table from a part
+                raise IndexUnavailable("has_more without hits or a cursor")
+        raise IndexUnavailable("the walk did not end")

services/explorer/dist/build-info.json

@@ -1,7 +1,7 @@
 {
   "product": "simready-central-slim-explorer",
-  "version": "slim-ui-e4522a3f",
-  "bundle_id": "main.01872f5b.js",
-  "built_at": "2026-10-18T17:30:52Z",
+  "version": "slim-ui-940a240b",
+  "bundle_id": "main.d09cd33e.js",
+  "built_at": "2026-10-19T08:19:49Z",
   "flags": {"public_mode": true, "email_auth": false, "create_ui": false, "hierarchy_editor": false, "create_search": false, "auth_ui": false, "credentials_ui": false, "index_management": false, "inference_backend": "hub", "vlm_asset_refs": false, "validation_image_max_px": 448, "validation_max_retries": 0, "validation_max_concurrent": 32, "validation_tail_concurrent": 8, "validation_eager_count": 50, "validation_batch_stream": true, "library_idle_prefetch": false, "validation_score_scale": "percent", "parse_gate_fields": true, "parse_fast_path": true, "featured_landing": true, "hf_download_commands": true, "dependency_views": true, "model_viewer": false, "library_viewer": true, "library_media": "runtime", "spell_correction": true, "downloads": "runtime", "submit_content": "runtime", "third_party_notices": "runtime"}
 }

services/explorer/dist/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/ds.png"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="SimReady Central — SimReady asset search"/><meta name="ui-version" content="slim-ui-e4522a3f"><script src="/config.js"></script><title>SimReady Central</title><script defer="defer" src="/static/js/main.01872f5b.js"></script><link href="/static/css/main.ed151f6a.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/ds.png"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="SimReady Central — SimReady asset search"/><meta name="ui-version" content="slim-ui-940a240b"><script src="/config.js"></script><title>SimReady Central</title><script defer="defer" src="/static/js/main.d09cd33e.js"></script><link href="/static/css/main.ed151f6a.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

services/explorer/dist/static/js/main.d09cd33e.js.LICENSE.txt

@@ -0,0 +1,39 @@
+/**
+ * @license React
+ * react-dom.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * @license React
+ * react-jsx-runtime.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * @license React
+ * react.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * @license React
+ * scheduler.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */

services/explorer/dist/third-party-notices.txt

@@ -1,4 +1,4 @@
-Third-party notices — SimReady Central Explorer slim-ui-e4522a3f
+Third-party notices — SimReady Central Explorer slim-ui-940a240b
 
 The software in this web application's browser bundle includes the packages below, each under its own
 license. Each entry gives the package, its version, its license and the license / notice text it ships.