Spaces:
Running on RTX PRO 6000
Running on RTX PRO 6000
Deploy indexing main 3afdae5 (run 20261020-a3cb7f2 kept)
Browse filescheck-image cache (#125), nginx upstream zone (#126), UI slim-ui-f57ff00c (curated titles, GB300 click-to-load). RESTORE_RUN_ID unchanged.
- Dockerfile +10 -0
- docker/hf-space/check_image_cache.pth +3 -0
- docker/hf-space/check_image_cache.py +487 -0
- docker/hf-space/nginx.conf.template +5 -1
- docker/hf-space/supervisord-gpu.conf +18 -0
- pipeline/config.py +8 -0
- pipeline/run_pipeline.py +35 -16
- pipeline/vision_views.py +126 -31
- pipeline/vision_views_worker.py +1 -0
- services/explorer/dist/build-info.json +4 -4
- services/explorer/dist/index.html +1 -1
- services/explorer/dist/static/css/{main.c80b163d.css β main.ed151f6a.css} +0 -0
- services/explorer/dist/static/js/main.2f1a816a.js +0 -0
- services/explorer/dist/static/js/main.e1d091ef.js +0 -0
- services/explorer/dist/static/js/{main.2f1a816a.js.LICENSE.txt β main.e1d091ef.js.LICENSE.txt} +0 -0
- services/explorer/dist/third-party-notices.txt +1 -1
Dockerfile
CHANGED
|
@@ -183,6 +183,16 @@ RUN python3.12 -m venv /app/.venv \
|
|
| 183 |
&& rm -rf /tmp/usdsearch-wheels \
|
| 184 |
&& chown -R 1000:1000 /app/.venv
|
| 185 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
# hf-storage-api β see docker/Dockerfile.hf-space's own copy of this block
|
| 187 |
# for the full explanation (identical here; this variant serves the same
|
| 188 |
# corpus/storage backend, just with a different SigLIP2 backend).
|
|
|
|
| 183 |
&& rm -rf /tmp/usdsearch-wheels \
|
| 184 |
&& chown -R 1000:1000 /app/.venv
|
| 185 |
|
| 186 |
+
# The vision checks' prepared asset views on disk (docker/hf-space/check_image_cache.py), on while the
|
| 187 |
+
# CHECK_IMAGE_CACHE_DIR Space Variable is set (pipeline/config.py). The .pth loads it in every venv python. The
|
| 188 |
+
# build fails unless it wraps the vendored llm_client and cached views come back byte-identical (--self-test).
|
| 189 |
+
COPY docker/hf-space/check_image_cache.py docker/hf-space/check_image_cache.pth /tmp/check-image-hook/
|
| 190 |
+
RUN SP=$(env -u PYTHONPATH /app/.venv/bin/python -c 'import sysconfig; print(sysconfig.get_path("purelib"))') \
|
| 191 |
+
&& install -m 644 /tmp/check-image-hook/check_image_cache.py /tmp/check-image-hook/check_image_cache.pth "$SP/" \
|
| 192 |
+
&& rm -rf /tmp/check-image-hook \
|
| 193 |
+
&& env -u PYTHONPATH CHECK_IMAGE_CACHE_DIR=/tmp/check-image-selftest /app/.venv/bin/python -m check_image_cache --self-test \
|
| 194 |
+
&& rm -rf /tmp/check-image-selftest
|
| 195 |
+
|
| 196 |
# hf-storage-api β see docker/Dockerfile.hf-space's own copy of this block
|
| 197 |
# for the full explanation (identical here; this variant serves the same
|
| 198 |
# corpus/storage backend, just with a different SigLIP2 backend).
|
docker/hf-space/check_image_cache.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:edc0d12a880df88947ee550690e3733aa0478b2e161ee4a566c6e62e78680051
|
| 3 |
+
size 99
|
docker/hf-space/check_image_cache.py
ADDED
|
@@ -0,0 +1,487 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The vision checks' prepared asset views, kept on disk for every deepsearch-api process of the Space.
|
| 2 |
+
|
| 3 |
+
Before each check, usdsearch's llm_client prepares the asset's stored view: it decodes the stored thumbnail, fits it
|
| 4 |
+
to USDSEARCH_VISION_VALIDATION_IMAGE_MAX_PX and re-encodes it as base64 PNG (``Validation._encode_reference_images``
|
| 5 |
+
-> ``_encode_image``). That is most of a check's API CPU, and a stored view never changes. With
|
| 6 |
+
``CHECK_IMAGE_CACHE_DIR`` set (a Space Variable, pipeline/config.py), each view ``_encode_reference_images`` prepares
|
| 7 |
+
is kept in ``<dir>/<settings>/<sha256 of the stored base64>.b64``: a hit reads the file; a miss prepares the view as
|
| 8 |
+
before and writes it (a temp file, then ``os.replace``), so the API processes share one copy. ``<settings>`` is a
|
| 9 |
+
digest of everything else that can shape a view: the validation config, llm_client's validation module, Pillow and
|
| 10 |
+
its codecs. Query images are never cached. Each file carries its length and CRC-32. Any error (unreadable,
|
| 11 |
+
unwritable, full disk, a damaged file) falls back to preparing the view, so a check never fails here, and a damaged
|
| 12 |
+
file is written again. The views stay under MAX_BYTES and never leave less than MIN_FREE on the disk, which the
|
| 13 |
+
Space's index, media and caches share: past either, new views are prepared per check, as without the cache.
|
| 14 |
+
|
| 15 |
+
Unset or empty, nothing is imported or installed and checks run exactly as usdsearch's own code.
|
| 16 |
+
check_image_cache.pth, in the venv's site-packages, imports this module in every /app/.venv python (PYTHONPATH is
|
| 17 |
+
unset for them) while the variable is set. The import only arms a hook that wraps llm_client's Validation when, and
|
| 18 |
+
if, the process imports it. A Validation without the wrapped methods (a future wheel) logs one warning and runs
|
| 19 |
+
unchanged.
|
| 20 |
+
|
| 21 |
+
``python -m check_image_cache`` fills the cache once the index is in (supervisord-gpu.conf's
|
| 22 |
+
[program:check-image-cache]): every image of the image cache index, loaded as a check loads it and prepared through
|
| 23 |
+
the same wrapped path, so its files are the ones a check would write. It first removes what an earlier start left
|
| 24 |
+
(other settings' directories, stale temp files). ``--self-test`` is the image build's check (Dockerfile.hf-space-gpu).
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
import functools
|
| 28 |
+
import hashlib
|
| 29 |
+
import json
|
| 30 |
+
import logging
|
| 31 |
+
import os
|
| 32 |
+
import re
|
| 33 |
+
import shutil
|
| 34 |
+
import sys
|
| 35 |
+
import threading
|
| 36 |
+
import time
|
| 37 |
+
import weakref
|
| 38 |
+
import zlib
|
| 39 |
+
|
| 40 |
+
ENV = "CHECK_IMAGE_CACHE_DIR"
|
| 41 |
+
TARGET = "llm_client.validation.validation"
|
| 42 |
+
MAX_BYTES = 2_000_000_000 # ~15k views at ~130 KB (448-px PNG); the Space's ~2,600 stored images take ~340 MB
|
| 43 |
+
MIN_FREE = 12_000_000_000 # above the 8 GB the media loader keeps free
|
| 44 |
+
RECHECK_BYTES, RECHECK_S = 64_000_000, 60 # each process re-reads the size and the free space this often
|
| 45 |
+
_DIGEST = re.compile(r"^[0-9a-f]{24}$")
|
| 46 |
+
_TMP = re.compile(r"^[0-9a-f]{64}\.b64\.\d+\.\d+\.tmp$")
|
| 47 |
+
_MARK = "__check_image_cache__"
|
| 48 |
+
logger = logging.getLogger("check_image_cache")
|
| 49 |
+
_warned = set()
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _warn_once(kind, message, *args):
|
| 53 |
+
if kind not in _warned:
|
| 54 |
+
_warned.add(kind)
|
| 55 |
+
logger.warning(message, *args)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def settings_digest(validation, module_file):
|
| 59 |
+
"""What, besides the stored image, shapes a prepared view: one directory per combination."""
|
| 60 |
+
try:
|
| 61 |
+
from PIL import __version__ as pillow, features
|
| 62 |
+
except ImportError: # no Pillow, no views to prepare (the tests' stand-in Validation)
|
| 63 |
+
pillow = features = None
|
| 64 |
+
config = validation._config
|
| 65 |
+
dump = config.model_dump() if hasattr(config, "model_dump") else dict(vars(config))
|
| 66 |
+
codecs = {}
|
| 67 |
+
for name in ("jpg", "zlib", "webp"):
|
| 68 |
+
try:
|
| 69 |
+
codecs[name] = features.version(name)
|
| 70 |
+
except Exception:
|
| 71 |
+
codecs[name] = None
|
| 72 |
+
with open(module_file, "rb") as fh:
|
| 73 |
+
module = hashlib.sha256(fh.read()).hexdigest()
|
| 74 |
+
identity = {
|
| 75 |
+
"config": dump, # all of it: a setting llm_client reads later can never share a stale directory
|
| 76 |
+
"class": f"{type(validation).__module__}.{type(validation).__qualname__}",
|
| 77 |
+
"module": module,
|
| 78 |
+
"pillow": pillow,
|
| 79 |
+
"codecs": codecs,
|
| 80 |
+
"python": "%d.%d" % sys.version_info[:2],
|
| 81 |
+
}
|
| 82 |
+
return hashlib.sha256(json.dumps(identity, sort_keys=True, default=str).encode()).hexdigest()[:24]
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class Cache:
|
| 86 |
+
"""One Validation's files, ``<root>/<settings digest>/<key>.b64``. Never raises: a failure is a miss."""
|
| 87 |
+
|
| 88 |
+
def __init__(self, directory):
|
| 89 |
+
self.directory = directory
|
| 90 |
+
self._lock = threading.Lock()
|
| 91 |
+
self.counts = {"hit": 0, "written": 0, "not_kept": 0, "bypass": 0}
|
| 92 |
+
self._used = self._free = None # bytes of views and free on the disk, as last read plus this process's writes
|
| 93 |
+
self._since, self._read_at = 0, 0.0
|
| 94 |
+
|
| 95 |
+
def _count(self, outcome):
|
| 96 |
+
with self._lock:
|
| 97 |
+
self.counts[outcome] += 1
|
| 98 |
+
|
| 99 |
+
def _room(self, size):
|
| 100 |
+
"""True while the views stay under MAX_BYTES and the disk keeps MIN_FREE: both re-read from the disk every
|
| 101 |
+
RECHECK_BYTES this process writes or RECHECK_S, so all processes together overshoot by little."""
|
| 102 |
+
with self._lock:
|
| 103 |
+
now = time.monotonic()
|
| 104 |
+
if self._used is None or self._since >= RECHECK_BYTES or now - self._read_at >= RECHECK_S:
|
| 105 |
+
self._used, self._free = files(self.directory)[1], _free(self.directory)
|
| 106 |
+
self._since, self._read_at = 0, now
|
| 107 |
+
if self._used + size > MAX_BYTES or self._free - size < MIN_FREE:
|
| 108 |
+
return False
|
| 109 |
+
self._used += size
|
| 110 |
+
self._free -= size
|
| 111 |
+
self._since += size
|
| 112 |
+
return True
|
| 113 |
+
|
| 114 |
+
def view(self, validation, image):
|
| 115 |
+
"""``validation._encode_image(image)``, read from its file when there is one."""
|
| 116 |
+
path = self._path(validation, image)
|
| 117 |
+
if path is None:
|
| 118 |
+
self._count("bypass")
|
| 119 |
+
return validation._encode_image(image)
|
| 120 |
+
value = self._read(path)
|
| 121 |
+
if value is not None:
|
| 122 |
+
self._count("hit")
|
| 123 |
+
return value
|
| 124 |
+
value = validation._encode_image(image) # its errors are the check's, as without the cache
|
| 125 |
+
self._count("written" if self._write(path, value) else "not_kept")
|
| 126 |
+
return value
|
| 127 |
+
|
| 128 |
+
def _path(self, validation, image):
|
| 129 |
+
# Only stored base64 images: a str that is not one is a file path, whose content can change under its name.
|
| 130 |
+
try:
|
| 131 |
+
if isinstance(image, str) and validation._is_base64(image):
|
| 132 |
+
return os.path.join(self.directory, hashlib.sha256(image.encode()).hexdigest() + ".b64")
|
| 133 |
+
except Exception as e:
|
| 134 |
+
_warn_once("key", "check image cache: no key for an image (%s); preparing it per check", e)
|
| 135 |
+
return None
|
| 136 |
+
|
| 137 |
+
def _read(self, path):
|
| 138 |
+
try:
|
| 139 |
+
with open(path, "rb") as fh:
|
| 140 |
+
data = fh.read()
|
| 141 |
+
except FileNotFoundError:
|
| 142 |
+
return None
|
| 143 |
+
except Exception as e:
|
| 144 |
+
_warn_once("read", "check image cache: cannot read %s (%s); preparing views per check", path, e)
|
| 145 |
+
return None
|
| 146 |
+
head, _, body = data.partition(b"\n")
|
| 147 |
+
try:
|
| 148 |
+
size, crc = head.split(b" ")
|
| 149 |
+
if int(size) == len(body) and int(crc, 16) == zlib.crc32(body):
|
| 150 |
+
return body.decode("ascii")
|
| 151 |
+
except (ValueError, UnicodeDecodeError):
|
| 152 |
+
pass
|
| 153 |
+
_warn_once("damaged", "check image cache: %s is damaged; preparing the view again", path)
|
| 154 |
+
return None
|
| 155 |
+
|
| 156 |
+
def _write(self, path, value):
|
| 157 |
+
tmp = f"{path}.{os.getpid()}.{threading.get_ident()}.tmp"
|
| 158 |
+
try:
|
| 159 |
+
body = value.encode("ascii")
|
| 160 |
+
if not self._room(len(body) + 32):
|
| 161 |
+
_warn_once("full", "check image cache: full (%d MB of views, or under %d GB free on the disk); "
|
| 162 |
+
"new views are prepared per check", MAX_BYTES // 1_000_000, MIN_FREE // 1_000_000_000)
|
| 163 |
+
return False
|
| 164 |
+
try:
|
| 165 |
+
fh = open(tmp, "wb")
|
| 166 |
+
except FileNotFoundError:
|
| 167 |
+
os.makedirs(self.directory, exist_ok=True)
|
| 168 |
+
fh = open(tmp, "wb")
|
| 169 |
+
with fh:
|
| 170 |
+
fh.write(b"%d %08x\n" % (len(body), zlib.crc32(body)))
|
| 171 |
+
fh.write(body)
|
| 172 |
+
os.replace(tmp, path)
|
| 173 |
+
return True
|
| 174 |
+
except Exception as e:
|
| 175 |
+
_warn_once("write", "check image cache: cannot write in %s (%s); views are prepared per check",
|
| 176 |
+
self.directory, e)
|
| 177 |
+
try:
|
| 178 |
+
os.unlink(tmp)
|
| 179 |
+
except OSError:
|
| 180 |
+
pass
|
| 181 |
+
return False
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
_caches = weakref.WeakKeyDictionary()
|
| 185 |
+
_caches_lock = threading.Lock()
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def is_wrapped(cls):
|
| 189 |
+
return getattr(getattr(cls, "_encode_reference_images", None), _MARK, None) is not None
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def cache_of(validation):
|
| 193 |
+
"""The Cache of a Validation whose class is wrapped, else None (also when its settings can't be read)."""
|
| 194 |
+
mark = getattr(getattr(type(validation), "_encode_reference_images", None), _MARK, None)
|
| 195 |
+
if mark is None:
|
| 196 |
+
return None
|
| 197 |
+
try:
|
| 198 |
+
return _caches[validation]
|
| 199 |
+
except KeyError:
|
| 200 |
+
pass
|
| 201 |
+
except TypeError: # not weak-referenceable: no cache
|
| 202 |
+
return None
|
| 203 |
+
root, module_file = mark
|
| 204 |
+
try:
|
| 205 |
+
cache = Cache(os.path.join(root, settings_digest(validation, module_file)))
|
| 206 |
+
except Exception as e:
|
| 207 |
+
_warn_once("settings", "check image cache off: cannot read the view settings (%s)", e)
|
| 208 |
+
cache = None
|
| 209 |
+
with _caches_lock:
|
| 210 |
+
return _caches.setdefault(validation, cache)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def patch(module, root):
|
| 214 |
+
"""Wrap ``module.Validation._encode_reference_images`` (llm_client's validation module); True when wrapped."""
|
| 215 |
+
cls = getattr(module, "Validation", None)
|
| 216 |
+
original = getattr(cls, "_encode_reference_images", None)
|
| 217 |
+
if original is None or not all(callable(getattr(cls, n, None)) for n in ("_encode_image", "_is_base64")):
|
| 218 |
+
_warn_once("missing", "check image cache off: %s.Validation has no _encode_reference_images, "
|
| 219 |
+
"_encode_image or _is_base64", getattr(module, "__name__", module))
|
| 220 |
+
return False
|
| 221 |
+
if getattr(original, _MARK, None) is not None:
|
| 222 |
+
return True
|
| 223 |
+
|
| 224 |
+
@functools.wraps(original)
|
| 225 |
+
def _encode_reference_images(self, reference_images):
|
| 226 |
+
cache = cache_of(self) if reference_images else None
|
| 227 |
+
if cache is None:
|
| 228 |
+
return original(self, reference_images)
|
| 229 |
+
return [cache.view(self, image) for image in reference_images]
|
| 230 |
+
|
| 231 |
+
setattr(_encode_reference_images, _MARK, (root, getattr(module, "__file__", None)))
|
| 232 |
+
cls._encode_reference_images = _encode_reference_images
|
| 233 |
+
return True
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
class _PatchOnImport:
|
| 237 |
+
"""A sys.meta_path finder: wraps llm_client's Validation right after its module has run, in the process that
|
| 238 |
+
imports it. Every other import passes through untouched."""
|
| 239 |
+
|
| 240 |
+
def __init__(self, root):
|
| 241 |
+
self.root = root
|
| 242 |
+
|
| 243 |
+
def find_spec(self, name, path, target=None):
|
| 244 |
+
if name != TARGET:
|
| 245 |
+
return None
|
| 246 |
+
try:
|
| 247 |
+
from importlib.machinery import PathFinder, SourceFileLoader, SourcelessFileLoader
|
| 248 |
+
|
| 249 |
+
spec = PathFinder.find_spec(name, path)
|
| 250 |
+
if spec is None:
|
| 251 |
+
return None
|
| 252 |
+
# A file loader serves this one module, so its exec_module can be wrapped in place.
|
| 253 |
+
if type(spec.loader) not in (SourceFileLoader, SourcelessFileLoader):
|
| 254 |
+
_warn_once("hook", "check image cache off: %s is not loaded from a file", name)
|
| 255 |
+
return None
|
| 256 |
+
exec_module = spec.loader.exec_module
|
| 257 |
+
root = self.root
|
| 258 |
+
|
| 259 |
+
def exec_and_patch(module):
|
| 260 |
+
exec_module(module)
|
| 261 |
+
try:
|
| 262 |
+
patch(module, root)
|
| 263 |
+
except Exception as e:
|
| 264 |
+
_warn_once("patch", "check image cache off: %s", e)
|
| 265 |
+
|
| 266 |
+
spec.loader.exec_module = exec_and_patch
|
| 267 |
+
return spec
|
| 268 |
+
except Exception as e:
|
| 269 |
+
_warn_once("hook", "check image cache off: %s", e)
|
| 270 |
+
return None
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def install(environ=os.environ):
|
| 274 |
+
"""Arm the hook when CHECK_IMAGE_CACHE_DIR is set; with it unset or empty, nothing changes."""
|
| 275 |
+
root = (environ.get(ENV) or "").strip()
|
| 276 |
+
if not root:
|
| 277 |
+
return False
|
| 278 |
+
module = sys.modules.get(TARGET)
|
| 279 |
+
if module is not None:
|
| 280 |
+
return patch(module, root)
|
| 281 |
+
if not any(isinstance(f, _PatchOnImport) for f in sys.meta_path):
|
| 282 |
+
sys.meta_path.insert(0, _PatchOnImport(root))
|
| 283 |
+
return True
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
async def fill(validation, loader, threads=4, page=500, batch=25):
|
| 287 |
+
"""Prepare every image of the loader's index through ``validation._encode_reference_images``.
|
| 288 |
+
|
| 289 |
+
The ids come from a scroll of the index, the images from ``loader.load_images`` (what a check loads), and up to
|
| 290 |
+
``threads`` views are prepared at once in threads. Returns {"images", "empty", "failed"}.
|
| 291 |
+
"""
|
| 292 |
+
import asyncio
|
| 293 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 294 |
+
|
| 295 |
+
counts = {"images": 0, "empty": 0, "failed": 0}
|
| 296 |
+
loop = asyncio.get_running_loop()
|
| 297 |
+
slots = asyncio.Semaphore(threads * 2)
|
| 298 |
+
pending = set()
|
| 299 |
+
|
| 300 |
+
def done(task): # on the loop's thread
|
| 301 |
+
slots.release()
|
| 302 |
+
pending.discard(task)
|
| 303 |
+
if task.cancelled() or task.exception() is not None:
|
| 304 |
+
counts["failed"] += 1
|
| 305 |
+
|
| 306 |
+
with ThreadPoolExecutor(threads, thread_name_prefix="check-image-cache") as pool:
|
| 307 |
+
client, index = loader.client, loader.image_cache_index
|
| 308 |
+
resp = await client.search(index=index, body={"query": {"match_all": {}}, "_source": False, "sort": ["_doc"]},
|
| 309 |
+
size=page, scroll="5m")
|
| 310 |
+
try:
|
| 311 |
+
while resp["hits"]["hits"]:
|
| 312 |
+
ids = [hit["_id"] for hit in resp["hits"]["hits"]]
|
| 313 |
+
for i in range(0, len(ids), batch):
|
| 314 |
+
images = await loader.load_images(ids[i:i + batch])
|
| 315 |
+
for image_id in ids[i:i + batch]:
|
| 316 |
+
image = images.get(image_id)
|
| 317 |
+
if not isinstance(image, str) or not image:
|
| 318 |
+
counts["empty"] += 1
|
| 319 |
+
continue
|
| 320 |
+
counts["images"] += 1
|
| 321 |
+
await slots.acquire()
|
| 322 |
+
task = loop.run_in_executor(pool, validation._encode_reference_images, [image])
|
| 323 |
+
pending.add(task)
|
| 324 |
+
task.add_done_callback(done)
|
| 325 |
+
resp = await client.scroll(scroll_id=resp["_scroll_id"], scroll="5m")
|
| 326 |
+
finally:
|
| 327 |
+
if pending:
|
| 328 |
+
await asyncio.gather(*pending, return_exceptions=True)
|
| 329 |
+
try:
|
| 330 |
+
await client.clear_scroll(scroll_id=resp["_scroll_id"])
|
| 331 |
+
except Exception:
|
| 332 |
+
pass
|
| 333 |
+
return counts
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def files(directory):
|
| 337 |
+
"""(count, bytes) of the views in a settings directory."""
|
| 338 |
+
n = size = 0
|
| 339 |
+
try:
|
| 340 |
+
with os.scandir(directory) as it:
|
| 341 |
+
for entry in it:
|
| 342 |
+
if entry.name.endswith(".b64"):
|
| 343 |
+
try:
|
| 344 |
+
size += entry.stat().st_size
|
| 345 |
+
n += 1
|
| 346 |
+
except OSError: # replaced or removed meanwhile
|
| 347 |
+
pass
|
| 348 |
+
except OSError:
|
| 349 |
+
pass
|
| 350 |
+
return n, size
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def _free(path):
|
| 354 |
+
"""Free bytes on the disk that holds ``path`` (its nearest existing parent); 0 when unreadable: no writes."""
|
| 355 |
+
try:
|
| 356 |
+
while not os.path.exists(path) and os.path.dirname(path) != path:
|
| 357 |
+
path = os.path.dirname(path)
|
| 358 |
+
return shutil.disk_usage(path).free
|
| 359 |
+
except OSError:
|
| 360 |
+
return 0
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
def sweep(directory, older_than=600):
|
| 364 |
+
"""What an earlier start left in the cache's root: other settings' directories, and temp files of writers that
|
| 365 |
+
died (older than ``older_than`` s). Only names this module makes are touched. Returns the number removed."""
|
| 366 |
+
root, removed = os.path.dirname(directory), 0
|
| 367 |
+
try:
|
| 368 |
+
with os.scandir(root) as it:
|
| 369 |
+
for entry in it:
|
| 370 |
+
if _DIGEST.match(entry.name) and entry.path != directory and entry.is_dir(follow_symlinks=False):
|
| 371 |
+
shutil.rmtree(entry.path, ignore_errors=True)
|
| 372 |
+
removed += 1
|
| 373 |
+
with os.scandir(directory) as it:
|
| 374 |
+
for entry in it:
|
| 375 |
+
if _TMP.match(entry.name) and time.time() - entry.stat().st_mtime > older_than:
|
| 376 |
+
os.unlink(entry.path)
|
| 377 |
+
removed += 1
|
| 378 |
+
except OSError:
|
| 379 |
+
pass
|
| 380 |
+
return removed
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
def self_test():
|
| 384 |
+
"""The image build's check (Dockerfile.hf-space-gpu): the .pth loaded this module and it wrapped the vendored
|
| 385 |
+
llm_client, views read back from the cache are byte-identical to llm_client's own, and the warm-up's imports
|
| 386 |
+
exist. Raises on any difference. Writes only in CHECK_IMAGE_CACHE_DIR, which the build then removes."""
|
| 387 |
+
import base64
|
| 388 |
+
import io
|
| 389 |
+
|
| 390 |
+
from PIL import Image
|
| 391 |
+
from deepsearch_api.routers_v2.dependencies import build_image_loader # noqa: F401 (the warm-up's imports)
|
| 392 |
+
from deepsearch_api.search_backend.main import SearchSettings # noqa: F401
|
| 393 |
+
from deepsearch_api.validation.config import ValidationSettings
|
| 394 |
+
from llm_client.validation.validation import Validation
|
| 395 |
+
|
| 396 |
+
assert "check_image_cache" in sys.modules and is_wrapped(Validation), "the hook did not wrap llm_client"
|
| 397 |
+
global MIN_FREE
|
| 398 |
+
MIN_FREE = 0 # the image builder's disk is not the Space's
|
| 399 |
+
|
| 400 |
+
def b64(image, fmt, **kw):
|
| 401 |
+
buf = io.BytesIO()
|
| 402 |
+
image.save(buf, fmt, **kw)
|
| 403 |
+
return base64.b64encode(buf.getvalue()).decode()
|
| 404 |
+
|
| 405 |
+
noise = Image.effect_noise((1024, 1024), 40)
|
| 406 |
+
rgb = Image.merge("RGB", (noise, Image.linear_gradient("L").resize((1024, 1024)), noise.rotate(90)))
|
| 407 |
+
rgba = rgb.resize((900, 600))
|
| 408 |
+
rgba.putalpha(Image.linear_gradient("L").resize((900, 600)))
|
| 409 |
+
images = [b64(rgb, "JPEG", quality=90), b64(rgba, "PNG"), b64(rgb.resize((300, 200)), "JPEG", quality=90)]
|
| 410 |
+
config = ValidationSettings(image_max_px=448).to_validation_config()
|
| 411 |
+
plain = [Validation(config=config)._encode_image(image) for image in images]
|
| 412 |
+
miss = Validation(config=config)._encode_reference_images(images)
|
| 413 |
+
reader = Validation(config=config)
|
| 414 |
+
hit = reader._encode_reference_images(images)
|
| 415 |
+
assert miss == plain and hit == plain, "a cached view differs from llm_client's own"
|
| 416 |
+
assert cache_of(reader).counts["hit"] == len(images), "the views were not read back from the cache"
|
| 417 |
+
return len(images)
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
def main(argv=None):
|
| 421 |
+
"""``python -m check_image_cache [--threads N]``: fill the cache from the image cache index. Always exits 0
|
| 422 |
+
(``--self-test`` exits 1 when the check fails)."""
|
| 423 |
+
import argparse
|
| 424 |
+
import asyncio
|
| 425 |
+
import resource
|
| 426 |
+
|
| 427 |
+
parser = argparse.ArgumentParser(prog="check_image_cache")
|
| 428 |
+
parser.add_argument("--threads", type=int, default=4)
|
| 429 |
+
parser.add_argument("--self-test", action="store_true")
|
| 430 |
+
args = parser.parse_args(argv)
|
| 431 |
+
say = functools.partial(print, "check-image-cache:", flush=True)
|
| 432 |
+
if not (os.environ.get(ENV) or "").strip():
|
| 433 |
+
say(f"{ENV} is not set: nothing to do")
|
| 434 |
+
return 1 if args.self_test else 0
|
| 435 |
+
if args.self_test:
|
| 436 |
+
say(f"self-test passed: {self_test()} views read back byte-identical")
|
| 437 |
+
return 0
|
| 438 |
+
started, cpu = time.monotonic(), time.process_time()
|
| 439 |
+
try:
|
| 440 |
+
from deepsearch_api.validation.config import ValidationSettings
|
| 441 |
+
|
| 442 |
+
settings = ValidationSettings()
|
| 443 |
+
if not settings.enabled:
|
| 444 |
+
say("vision validation is off: nothing to do")
|
| 445 |
+
return 0
|
| 446 |
+
from llm_client.validation.validation import Validation
|
| 447 |
+
|
| 448 |
+
validation = Validation(config=settings.to_validation_config())
|
| 449 |
+
cache = cache_of(validation)
|
| 450 |
+
if cache is None:
|
| 451 |
+
say("llm_client's Validation is not wrapped (see the warning above): nothing to do")
|
| 452 |
+
return 0
|
| 453 |
+
swept = sweep(cache.directory)
|
| 454 |
+
from deepsearch_api.routers_v2.dependencies import build_image_loader
|
| 455 |
+
from deepsearch_api.search_backend.main import SearchSettings
|
| 456 |
+
|
| 457 |
+
loader = build_image_loader(SearchSettings())
|
| 458 |
+
|
| 459 |
+
async def run():
|
| 460 |
+
try:
|
| 461 |
+
return await fill(validation, loader, threads=max(1, args.threads))
|
| 462 |
+
finally:
|
| 463 |
+
await loader.client.close()
|
| 464 |
+
|
| 465 |
+
counts = asyncio.run(run())
|
| 466 |
+
except Exception as e:
|
| 467 |
+
say(f"stopped ({type(e).__name__}: {e}); checks prepare the rest as they go")
|
| 468 |
+
return 0
|
| 469 |
+
n, size = files(cache.directory)
|
| 470 |
+
c = cache.counts
|
| 471 |
+
say(f"{counts['images']} images in {time.monotonic() - started:.1f} s, {time.process_time() - cpu:.1f} CPU-s: "
|
| 472 |
+
f"{c['written']} written, {c['hit']} already there, {c['not_kept']} not kept, {c['bypass']} not cacheable, "
|
| 473 |
+
f"{counts['failed']} failed, {counts['empty']} missing or not loaded; {n} files, {size / 1e6:.0f} MB in "
|
| 474 |
+
f"{cache.directory} ({swept} old entries removed); "
|
| 475 |
+
f"peak RSS {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024:.0f} MB")
|
| 476 |
+
return 0
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
if __name__ == "__main__":
|
| 480 |
+
import check_image_cache # the instance the .pth imported: its hook wrapped Validation
|
| 481 |
+
|
| 482 |
+
sys.exit(check_image_cache.main())
|
| 483 |
+
else:
|
| 484 |
+
try:
|
| 485 |
+
install()
|
| 486 |
+
except Exception as e: # never break a python start
|
| 487 |
+
_warn_once("install", "check image cache off: %s", e)
|
docker/hf-space/nginx.conf.template
CHANGED
|
@@ -107,9 +107,13 @@ map $http_accept_encoding $slim_accepts_br {
|
|
| 107 |
# deepsearch-api's processes, one uvicorn worker each. least_conn, not `uvicorn --workers N`: with N workers on one
|
| 108 |
# socket the kernel hands nearly every connection to 2-3 of them, and the rest of the CPU sits idle (lab load test
|
| 109 |
# 2026-10-15, docs/search-performance.md). __DEEPSEARCH_API_SERVERS__ is rendered at container start from
|
| 110 |
-
# DEEPSEARCH_API_PROCESSES (the nginx program in supervisord*.conf).
|
|
|
|
|
|
|
|
|
|
| 111 |
upstream deepsearch {
|
| 112 |
least_conn;
|
|
|
|
| 113 |
__DEEPSEARCH_API_SERVERS__
|
| 114 |
}
|
| 115 |
server {
|
|
|
|
| 107 |
# deepsearch-api's processes, one uvicorn worker each. least_conn, not `uvicorn --workers N`: with N workers on one
|
| 108 |
# socket the kernel hands nearly every connection to 2-3 of them, and the rest of the CPU sits idle (lab load test
|
| 109 |
# 2026-10-15, docs/search-performance.md). __DEEPSEARCH_API_SERVERS__ is rendered at container start from
|
| 110 |
+
# DEEPSEARCH_API_PROCESSES (the nginx program in supervisord*.conf). The zone shares least_conn's counts between
|
| 111 |
+
# nginx's workers: without it each worker balances only its own connections, so two searches' 50-check batches
|
| 112 |
+
# often land on one process and split its 16 check slots while others idle (lab 2026-10-20: p95 of all badges
|
| 113 |
+
# 4.0 -> 3.2 s at 20 users, 6.5 -> 5.4 s at 44).
|
| 114 |
upstream deepsearch {
|
| 115 |
least_conn;
|
| 116 |
+
zone deepsearch 64k;
|
| 117 |
__DEEPSEARCH_API_SERVERS__
|
| 118 |
}
|
| 119 |
server {
|
docker/hf-space/supervisord-gpu.conf
CHANGED
|
@@ -182,6 +182,24 @@ stdout_logfile_maxbytes=0
|
|
| 182 |
stderr_logfile=/dev/stderr
|
| 183 |
stderr_logfile_maxbytes=0
|
| 184 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 185 |
[program:deepsearch-api]
|
| 186 |
; 6 processes with ONE uvicorn worker each, on 127.0.0.1:8101-8106, behind
|
| 187 |
; nginx's least_conn upstream (nginx.conf.template). Not `uvicorn --workers 6`:
|
|
|
|
| 182 |
stderr_logfile=/dev/stderr
|
| 183 |
stderr_logfile_maxbytes=0
|
| 184 |
|
| 185 |
+
[program:check-image-cache]
|
| 186 |
+
; Fills CHECK_IMAGE_CACHE_DIR (a Space Variable) with every stored image's prepared check view:
|
| 187 |
+
; docker/hf-space/check_image_cache.py. Starts once the Space serves (nginx's /health): started with the API
|
| 188 |
+
; processes, it held their start back by ~5 s in the lab. They read and fill the same files on their own, so
|
| 189 |
+
; nothing waits on this program. Niced, 4 threads; it always exits 0 and is never restarted. Without the
|
| 190 |
+
; variable it does nothing.
|
| 191 |
+
command=/usr/bin/env -u PYTHONPATH /bin/sh -c "until curl -sf http://localhost:%(ENV_PORT)s/health >/dev/null 2>&1; do sleep 5; done; cd /app && exec nice -n 10 /app/.venv/bin/python -m check_image_cache --threads 4"
|
| 192 |
+
directory=/app
|
| 193 |
+
user=1000
|
| 194 |
+
autorestart=false
|
| 195 |
+
startsecs=0
|
| 196 |
+
startretries=0
|
| 197 |
+
exitcodes=0
|
| 198 |
+
stdout_logfile=/dev/stdout
|
| 199 |
+
stdout_logfile_maxbytes=0
|
| 200 |
+
stderr_logfile=/dev/stderr
|
| 201 |
+
stderr_logfile_maxbytes=0
|
| 202 |
+
|
| 203 |
[program:deepsearch-api]
|
| 204 |
; 6 processes with ONE uvicorn worker each, on 127.0.0.1:8101-8106, behind
|
| 205 |
; nginx's least_conn upstream (nginx.conf.template). Not `uvicorn --workers 6`:
|
pipeline/config.py
CHANGED
|
@@ -127,6 +127,9 @@ class Config:
|
|
| 127 |
]:
|
| 128 |
if os.environ.get(key):
|
| 129 |
values[key] = os.environ[key]
|
|
|
|
|
|
|
|
|
|
| 130 |
return cls(values=values, env_file=path)
|
| 131 |
|
| 132 |
# ββ typed accessors βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -416,6 +419,11 @@ class Config:
|
|
| 416 |
# a text query's judge reads each hit's path (below its dataset or Isaac content root) and vision
|
| 417 |
# caption, and an image + words query asks whether the candidate matches both (usdsearch !279).
|
| 418 |
"USDSEARCH_VISION_VALIDATION_IMAGE_MAX_PX": "448",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 419 |
"USDSEARCH_VISION_VALIDATION_LOGPROB_ASSET_FACTS": json.dumps(["path", "vlm_metadata_caption"]),
|
| 420 |
"USDSEARCH_VISION_VALIDATION_LOGPROB_ASSET_PATH_ROOTS": json.dumps(VALIDATION_ASSET_PATH_ROOTS),
|
| 421 |
"USDSEARCH_VISION_VALIDATION_LOGPROB_IMAGE_TEXT_QUERY": "true",
|
|
|
|
| 127 |
]:
|
| 128 |
if os.environ.get(key):
|
| 129 |
values[key] = os.environ[key]
|
| 130 |
+
# the vision-views stage is run one-off from the shell (VISION_VIEWS=all VISION_VIEWS_APPLY=... python -m
|
| 131 |
+
# pipeline.run_pipeline vision-views): its settings win even when the env file does not name them
|
| 132 |
+
values.update({k: v for k, v in os.environ.items() if k.startswith("VISION_VIEWS") and v})
|
| 133 |
return cls(values=values, env_file=path)
|
| 134 |
|
| 135 |
# ββ typed accessors βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 419 |
# a text query's judge reads each hit's path (below its dataset or Isaac content root) and vision
|
| 420 |
# caption, and an image + words query asks whether the candidate matches both (usdsearch !279).
|
| 421 |
"USDSEARCH_VISION_VALIDATION_IMAGE_MAX_PX": "448",
|
| 422 |
+
# Each stored view is fitted and encoded for its check once per container, not once per check (most
|
| 423 |
+
# of a check's API CPU): docker/hf-space/check_image_cache.py, GPU image only (the CPU image has no
|
| 424 |
+
# hook: the variable does nothing there). Off: delete the Space Variable and restart; a deploy sets it
|
| 425 |
+
# again while this line is here (deploys never delete variables).
|
| 426 |
+
"CHECK_IMAGE_CACHE_DIR": "/tmp/check-image-cache",
|
| 427 |
"USDSEARCH_VISION_VALIDATION_LOGPROB_ASSET_FACTS": json.dumps(["path", "vlm_metadata_caption"]),
|
| 428 |
"USDSEARCH_VISION_VALIDATION_LOGPROB_ASSET_PATH_ROOTS": json.dumps(VALIDATION_ASSET_PATH_ROOTS),
|
| 429 |
"USDSEARCH_VISION_VALIDATION_LOGPROB_IMAGE_TEXT_QUERY": "true",
|
pipeline/run_pipeline.py
CHANGED
|
@@ -7,7 +7,8 @@ Stages run in order and each is independently re-runnable:
|
|
| 7 |
enumerate corpus.json -> exact asset list (N) + manifest
|
| 8 |
index bring the stack up, restore the baseline, clear, submit the
|
| 9 |
delta, wait, then the image-overrides, categories and
|
| 10 |
-
profiles steps
|
|
|
|
| 11 |
export elasticdump the indexes; neo4j-admin dump the graph
|
| 12 |
upload publish the artifact set to S3
|
| 13 |
bake build images with the data pre-loaded
|
|
@@ -55,7 +56,7 @@ import os
|
|
| 55 |
import subprocess
|
| 56 |
import sys
|
| 57 |
from pathlib import Path
|
| 58 |
-
from typing import List
|
| 59 |
|
| 60 |
from .config import BUILD_DIR, Config
|
| 61 |
|
|
@@ -155,15 +156,25 @@ def _mark_done(step: str, args) -> None:
|
|
| 155 |
(STAGE_MARKERS / f"{step}.done").touch()
|
| 156 |
|
| 157 |
|
| 158 |
-
def
|
| 159 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
report = BUILD_DIR / "index-report.json"
|
| 161 |
waited = report.stat().st_mtime if report.exists() else 0.0
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
|
|
|
|
|
|
|
|
|
| 167 |
|
| 168 |
|
| 169 |
def stage_image_overrides(cfg: Config, args) -> int:
|
|
@@ -191,13 +202,18 @@ def stage_profiles(cfg: Config, args) -> int:
|
|
| 191 |
|
| 192 |
|
| 193 |
def stage_vision_views(cfg: Config, args) -> int:
|
| 194 |
-
from .vision_views import run
|
| 195 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
|
| 197 |
|
| 198 |
def stage_index(cfg: Config, args) -> int:
|
| 199 |
-
"""Bring the stack up, restore the baseline, clear, submit the delta, wait, write
|
| 200 |
-
validation profiles the packages' sidecars name."""
|
| 201 |
steps = (
|
| 202 |
("up", stage_up),
|
| 203 |
("restore", stage_restore),
|
|
@@ -207,7 +223,9 @@ def stage_index(cfg: Config, args) -> int:
|
|
| 207 |
("image-overrides", stage_image_overrides),
|
| 208 |
("categories", stage_categories),
|
| 209 |
("profiles", stage_profiles),
|
| 210 |
-
|
|
|
|
|
|
|
| 211 |
)
|
| 212 |
for step, fn in steps:
|
| 213 |
print(f"\n{'=' * 70}\n== index: {step}\n{'=' * 70}")
|
|
@@ -219,11 +237,12 @@ def stage_index(cfg: Config, args) -> int:
|
|
| 219 |
|
| 220 |
|
| 221 |
def stage_export(cfg: Config, args) -> int:
|
| 222 |
-
missing = enrichment_missing()
|
| 223 |
if missing:
|
| 224 |
print(
|
| 225 |
f"refusing to export: {', '.join(missing)} not run since the last `wait`. They write into the index "
|
| 226 |
-
f"(image overrides, central:category, partner profiles). Run them in order,
|
|
|
|
| 227 |
+ "".join(f" python -m pipeline.run_pipeline {step} --env-file ...\n" for step in missing)
|
| 228 |
)
|
| 229 |
return 1
|
|
|
|
| 7 |
enumerate corpus.json -> exact asset list (N) + manifest
|
| 8 |
index bring the stack up, restore the baseline, clear, submit the
|
| 9 |
delta, wait, then the image-overrides, categories and
|
| 10 |
+
profiles steps and, last, vision-views when VISION_VIEWS is on
|
| 11 |
+
(HF prod: the committed captions, VISION_VIEWS_APPLY)
|
| 12 |
export elasticdump the indexes; neo4j-admin dump the graph
|
| 13 |
upload publish the artifact set to S3
|
| 14 |
bake build images with the data pre-loaded
|
|
|
|
| 56 |
import subprocess
|
| 57 |
import sys
|
| 58 |
from pathlib import Path
|
| 59 |
+
from typing import List, Optional
|
| 60 |
|
| 61 |
from .config import BUILD_DIR, Config
|
| 62 |
|
|
|
|
| 156 |
(STAGE_MARKERS / f"{step}.done").touch()
|
| 157 |
|
| 158 |
|
| 159 |
+
def _done_at(step: str) -> Optional[float]:
|
| 160 |
+
marker = STAGE_MARKERS / f"{step}.done"
|
| 161 |
+
return marker.stat().st_mtime if marker.exists() else None
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def enrichment_missing(cfg: Optional[Config] = None) -> List[str]:
|
| 165 |
+
"""The ENRICHMENT_STEPS that haven't succeeded since the latest `wait` (its .build/index-report.json), plus
|
| 166 |
+
`vision-views` when the config turns it on and it hasn't succeeded since the last of them: the captions go in
|
| 167 |
+
last (2026-10-20: a run shipped thumbnail captions when the manual captions step did not take effect)."""
|
| 168 |
report = BUILD_DIR / "index-report.json"
|
| 169 |
waited = report.stat().st_mtime if report.exists() else 0.0
|
| 170 |
+
done = {step: _done_at(step) for step in (*ENRICHMENT_STEPS, "vision-views")}
|
| 171 |
+
missing = [step for step in ENRICHMENT_STEPS if done[step] is None or done[step] < waited]
|
| 172 |
+
from .vision_views import is_on
|
| 173 |
+
if is_on(cfg):
|
| 174 |
+
last = max([waited] + [done[step] for step in ENRICHMENT_STEPS if done[step] is not None])
|
| 175 |
+
if done["vision-views"] is None or done["vision-views"] < last:
|
| 176 |
+
missing.append("vision-views")
|
| 177 |
+
return missing
|
| 178 |
|
| 179 |
|
| 180 |
def stage_image_overrides(cfg: Config, args) -> int:
|
|
|
|
| 202 |
|
| 203 |
|
| 204 |
def stage_vision_views(cfg: Config, args) -> int:
|
| 205 |
+
from .vision_views import is_on, run
|
| 206 |
+
code = run(cfg, dry_run=args.dry_run)
|
| 207 |
+
# Only a run that applied or generated captions counts: an off run (VISION_VIEWS=off for one command) must not
|
| 208 |
+
# let export ship the index without them.
|
| 209 |
+
if code == 0 and is_on(cfg):
|
| 210 |
+
_mark_done("vision-views", args)
|
| 211 |
+
return code
|
| 212 |
|
| 213 |
|
| 214 |
def stage_index(cfg: Config, args) -> int:
|
| 215 |
+
"""Bring the stack up, restore the baseline, clear, submit the delta, wait, write the image overrides,
|
| 216 |
+
central:category and the validation profiles the packages' sidecars name, then the captions."""
|
| 217 |
steps = (
|
| 218 |
("up", stage_up),
|
| 219 |
("restore", stage_restore),
|
|
|
|
| 223 |
("image-overrides", stage_image_overrides),
|
| 224 |
("categories", stage_categories),
|
| 225 |
("profiles", stage_profiles),
|
| 226 |
+
# last step that writes the index, right before `export`: image-overrides re-captions its assets, so the
|
| 227 |
+
# captions go after it (HF prod: VISION_VIEWS_APPLY, the committed captions). Off unless VISION_VIEWS=new|all
|
| 228 |
+
("vision-views", stage_vision_views),
|
| 229 |
)
|
| 230 |
for step, fn in steps:
|
| 231 |
print(f"\n{'=' * 70}\n== index: {step}\n{'=' * 70}")
|
|
|
|
| 237 |
|
| 238 |
|
| 239 |
def stage_export(cfg: Config, args) -> int:
|
| 240 |
+
missing = enrichment_missing(cfg)
|
| 241 |
if missing:
|
| 242 |
print(
|
| 243 |
f"refusing to export: {', '.join(missing)} not run since the last `wait`. They write into the index "
|
| 244 |
+
f"(image overrides, central:category, partner profiles, captions; captions last). Run them in order, "
|
| 245 |
+
f"then export:\n"
|
| 246 |
+ "".join(f" python -m pipeline.run_pipeline {step} --env-file ...\n" for step in missing)
|
| 247 |
)
|
| 248 |
return 1
|
pipeline/vision_views.py
CHANGED
|
@@ -10,8 +10,9 @@ that plugin, inside the usdsearch image, with labelled views from the library me
|
|
| 10 |
a dedicated all-open still);
|
| 11 |
* no media at all β the shipped thumbnail, as before;
|
| 12 |
|
| 13 |
-
plus a name hint (file name and two folders
|
| 14 |
-
already allow.
|
|
|
|
| 15 |
|
| 16 |
Then a mismatch report: each asset's new `object_type` against its file-name words and its Q-code's Wikidata label,
|
| 17 |
`.build/vision-views-mismatch.{json,csv}`. Flagged assets are listed for review; nothing is relabelled.
|
|
@@ -25,16 +26,23 @@ VISION_VIEWS_MEDIA_DIR names a library media tree (manifest.json + <id>/<file>);
|
|
| 25 |
fetched like the Space does (pipeline/library_media.py). VISION_VIEWS_HINT=off drops the hint; VISION_VIEWS_MAX caps
|
| 26 |
the number of assets (a trial); VISION_VIEWS_CONCURRENCY (4), VISION_VIEWS_TIMEOUT_S (300), VISION_VIEWS_TRIES (3).
|
| 27 |
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
"""
|
| 33 |
|
| 34 |
from __future__ import annotations
|
| 35 |
|
| 36 |
import argparse
|
| 37 |
import csv
|
|
|
|
|
|
|
| 38 |
import json
|
| 39 |
import os
|
| 40 |
import re
|
|
@@ -42,7 +50,7 @@ import subprocess
|
|
| 42 |
import sys
|
| 43 |
from collections import Counter
|
| 44 |
from pathlib import Path
|
| 45 |
-
from typing import Dict, Iterable, List, Optional
|
| 46 |
|
| 47 |
from . import hf_addressing
|
| 48 |
from .vision_views_rules import agrees, compatible, path_words, words # noqa: F401 (words: the tests)
|
|
@@ -52,6 +60,7 @@ SPEC = BUILD_DIR / "vision-views-spec.json"
|
|
| 52 |
REPORT = BUILD_DIR / "vision-views.json"
|
| 53 |
MISMATCH_JSON = BUILD_DIR / "vision-views-mismatch.json"
|
| 54 |
MISMATCH_CSV = BUILD_DIR / "vision-views-mismatch.csv"
|
|
|
|
| 55 |
WORKER = Path(__file__).with_name("vision_views_worker.py")
|
| 56 |
HELPERS = Path(__file__).with_name("image_overrides_worker.py")
|
| 57 |
RULES = Path(__file__).with_name("vision_views_rules.py")
|
|
@@ -59,6 +68,7 @@ ON_DISAGREE = ("flag", "reference", "hold")
|
|
| 59 |
COMPOSE_PROJECT = "simready-indexing"
|
| 60 |
SERVICE = "omni-writer"
|
| 61 |
MODES = ("off", "new", "all")
|
|
|
|
| 62 |
SIDES = ("right", "back", "left")
|
| 63 |
QCODE_PROPERTY = "simready_metadata_qcode"
|
| 64 |
|
|
@@ -123,9 +133,29 @@ def view_plan(entry: Optional[Dict]) -> List[Dict]:
|
|
| 123 |
return views
|
| 124 |
|
| 125 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
def asset_hint(address: str) -> str:
|
| 127 |
-
"""The file name and its two nearest folders β never the host, owner or revision
|
| 128 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
|
| 130 |
|
| 131 |
def media_tree(cfg: Config) -> Optional[Path]:
|
|
@@ -181,38 +211,79 @@ def override_image_matches(rec: Dict, entry: Dict) -> bool:
|
|
| 181 |
return not sha or (path.is_file() and hashlib.sha256(path.read_bytes()).hexdigest() == sha)
|
| 182 |
|
| 183 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
def apply_plan(cfg: Config, mode: str, path: Path) -> Dict:
|
| 185 |
"""Precomputed metadata to write as it is (no model call): {address: the writer's plugin fields}, for each
|
| 186 |
selected asset the file has at the SAME pinned address (same bytes) with a writer record. Others keep what the
|
| 187 |
thumbnail plugin wrote. An image-override asset is written only from a record made from its override picture;
|
| 188 |
-
else it keeps the image-overrides stage's caption."""
|
| 189 |
from .image_overrides import load_spec
|
|
|
|
| 190 |
records = {}
|
| 191 |
-
for line in
|
| 192 |
if line.strip():
|
| 193 |
rec = json.loads(line)
|
| 194 |
if (rec.get("writer") or {}).get("plugin_fields"):
|
| 195 |
records[rec["address"]] = rec
|
| 196 |
overrides = load_spec()
|
| 197 |
-
assets, missing, wrong_image = {},
|
| 198 |
for address in select_addresses(cfg, mode, keep_curated=True):
|
| 199 |
rec = records.get(address)
|
| 200 |
if rec is None:
|
| 201 |
-
missing
|
| 202 |
continue
|
| 203 |
if address in overrides and not override_image_matches(rec, overrides[address]):
|
| 204 |
-
wrong_image
|
| 205 |
continue
|
| 206 |
assets[address] = {"apply": rec["writer"]["plugin_fields"], "writer_model": rec["writer"].get("model"),
|
| 207 |
"check_object_type": ((rec.get("check") or {}).get("meta") or {}).get("object_type"),
|
| 208 |
"models_agree": rec.get("agree")}
|
| 209 |
-
return {"index": cfg.main_index, "mode": "apply", "source": str(path), "
|
| 210 |
-
"not_precomputed": missing, "override_image_mismatch": wrong_image
|
|
|
|
| 211 |
|
| 212 |
|
| 213 |
def plan(cfg: Config, mode: str) -> Dict:
|
| 214 |
addresses = select_addresses(cfg, mode)
|
| 215 |
-
hint_on = cfg.get("VISION_VIEWS_HINT", "on").strip().lower() not in
|
| 216 |
# thumbnail: every asset from its shipped thumbnail + the hint (measured as good as the views on object_type,
|
| 217 |
# at a quarter of the image tokens); views: the RTX views where the asset has them
|
| 218 |
views_on = cfg.get("VISION_VIEWS_IMAGES", "views").strip().lower() != "thumbnail"
|
|
@@ -309,32 +380,52 @@ def qcode_labels(qcodes: Iterable[str]) -> Dict[str, Dict]:
|
|
| 309 |
|
| 310 |
|
| 311 |
def run(cfg: Config, dry_run: bool = False) -> int:
|
| 312 |
-
mode =
|
| 313 |
if mode not in MODES:
|
| 314 |
print(f"VISION_VIEWS={mode!r}: expected one of {', '.join(MODES)}")
|
| 315 |
return 1
|
| 316 |
if mode == "off":
|
| 317 |
print("vision views: off (VISION_VIEWS) β captions stay as the thumbnail plugin wrote them")
|
| 318 |
return 0
|
| 319 |
-
|
| 320 |
-
if
|
| 321 |
-
# write precomputed metadata (
|
| 322 |
-
|
| 323 |
-
|
|
|
|
|
|
|
| 324 |
return 1
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 330 |
if dry_run:
|
| 331 |
print("dry run: nothing written")
|
| 332 |
return 0
|
|
|
|
|
|
|
| 333 |
return run_worker(cfg, spec)
|
| 334 |
if not dry_run and not os.environ.get("USDSEARCH_LLM_API_KEY", "").strip():
|
| 335 |
print("VISION_VIEWS is on but USDSEARCH_LLM_API_KEY is not set in the environment")
|
| 336 |
return 1
|
| 337 |
check_model = cfg.get("VISION_VIEWS_CHECK_MODEL").strip()
|
|
|
|
| 338 |
on_disagree = cfg.get("VISION_VIEWS_ON_DISAGREE", "flag").strip().lower() or "flag"
|
| 339 |
if on_disagree not in ON_DISAGREE:
|
| 340 |
print(f"VISION_VIEWS_ON_DISAGREE={on_disagree!r}: expected one of {', '.join(ON_DISAGREE)}")
|
|
@@ -343,10 +434,12 @@ def run(cfg: Config, dry_run: bool = False) -> int:
|
|
| 343 |
kinds = Counter(len(a["views"]) for a in spec["assets"].values())
|
| 344 |
opens = sum(1 for a in spec["assets"].values() if any(v["label"] == "open" for v in a["views"]))
|
| 345 |
images = sum(max(1, len(a["views"])) for a in spec["assets"].values())
|
|
|
|
| 346 |
calls = len(spec["assets"]) * (2 if check_model else 1)
|
| 347 |
print(f"vision views ({mode}): {len(spec['assets'])} asset(s), media {spec['media']}; views per asset "
|
| 348 |
-
f"{dict(sorted(kinds.items()))} (0 = thumbnail); {opens} with an open view; {images} image(s); {
|
| 349 |
-
f"
|
|
|
|
| 350 |
if dry_run:
|
| 351 |
print("dry run: nothing written")
|
| 352 |
return 0
|
|
@@ -376,7 +469,9 @@ def run_worker(cfg: Config, spec: Dict) -> int:
|
|
| 376 |
"-e", "USDSEARCH_VISION_METADATA_YAML_FILEPATH=/app/vlm_metadata_fields.yaml",
|
| 377 |
"-v", f"{REPO_ROOT / 'config' / 'vlm_metadata_prompt.txt'}:/app/vlm_metadata_prompt.txt:ro",
|
| 378 |
"-e", "USDSEARCH_VISION_METADATA_PROMPT_FILEPATH=/app/vlm_metadata_prompt.txt",
|
| 379 |
-
"-e", "OPENSEARCH_URL_INTERNAL=http://opensearch:9200"
|
|
|
|
|
|
|
| 380 |
if spec["media"]:
|
| 381 |
cmd += ["-v", f"{spec['media']}:/media:ro"]
|
| 382 |
for var in passthrough:
|
|
|
|
| 10 |
a dedicated all-open still);
|
| 11 |
* no media at all β the shipped thumbnail, as before;
|
| 12 |
|
| 13 |
+
plus a name hint (file name and two folders; none when the file name is only an id or a hash), which the prompt
|
| 14 |
+
allows only to choose between readings the views already allow. This stage sends only that hint: the container's own
|
| 15 |
+
VLM_ASSET_HINT is forced off. Needs usdsearch with labelled views (the plugin's `view_labels` / `asset_hint`).
|
| 16 |
|
| 17 |
Then a mismatch report: each asset's new `object_type` against its file-name words and its Q-code's Wikidata label,
|
| 18 |
`.build/vision-views-mismatch.{json,csv}`. Flagged assets are listed for review; nothing is relabelled.
|
|
|
|
| 26 |
fetched like the Space does (pipeline/library_media.py). VISION_VIEWS_HINT=off drops the hint; VISION_VIEWS_MAX caps
|
| 27 |
the number of assets (a trial); VISION_VIEWS_CONCURRENCY (4), VISION_VIEWS_TIMEOUT_S (300), VISION_VIEWS_TRIES (3).
|
| 28 |
|
| 29 |
+
VISION_VIEWS_APPLY=<captions.jsonl[.gz]> writes captions made ahead of the run instead (no model call, any usdsearch
|
| 30 |
+
image): gzip read by suffix, checked against its `<name>.jsonl.sha256` sibling when there is one. HF prod's env file
|
| 31 |
+
names the committed file (config/vision_captions/), so every run applies it as the last step of `index`, before
|
| 32 |
+
export. Assets the file does not cover keep their caption and are listed in .build/vision-views-apply.json.
|
| 33 |
+
|
| 34 |
+
VISION_VIEWS_CHECK_MODEL (+ VISION_VIEWS_CHECK_REASONING_EFFORT) has a second model caption the same inputs; off,
|
| 35 |
+
none, false or 0 turns it off (the shell can override the env file's). On a disagreement VISION_VIEWS_ON_DISAGREE
|
| 36 |
+
writes: flag (default: the primary's; never the other model's) | reference (the name matching the file-name or Q-code
|
| 37 |
+
words when exactly one does, else the primary's) | hold (neither). Disagreements are flagged.
|
| 38 |
"""
|
| 39 |
|
| 40 |
from __future__ import annotations
|
| 41 |
|
| 42 |
import argparse
|
| 43 |
import csv
|
| 44 |
+
import gzip
|
| 45 |
+
import hashlib
|
| 46 |
import json
|
| 47 |
import os
|
| 48 |
import re
|
|
|
|
| 50 |
import sys
|
| 51 |
from collections import Counter
|
| 52 |
from pathlib import Path
|
| 53 |
+
from typing import Dict, Iterable, List, Optional, Tuple
|
| 54 |
|
| 55 |
from . import hf_addressing
|
| 56 |
from .vision_views_rules import agrees, compatible, path_words, words # noqa: F401 (words: the tests)
|
|
|
|
| 60 |
REPORT = BUILD_DIR / "vision-views.json"
|
| 61 |
MISMATCH_JSON = BUILD_DIR / "vision-views-mismatch.json"
|
| 62 |
MISMATCH_CSV = BUILD_DIR / "vision-views-mismatch.csv"
|
| 63 |
+
APPLY_LISTING = "vision-views-apply.json" # under BUILD_DIR: what an apply wrote and what it left as it was
|
| 64 |
WORKER = Path(__file__).with_name("vision_views_worker.py")
|
| 65 |
HELPERS = Path(__file__).with_name("image_overrides_worker.py")
|
| 66 |
RULES = Path(__file__).with_name("vision_views_rules.py")
|
|
|
|
| 68 |
COMPOSE_PROJECT = "simready-indexing"
|
| 69 |
SERVICE = "omni-writer"
|
| 70 |
MODES = ("off", "new", "all")
|
| 71 |
+
OFF = ("off", "none", "false", "0", "no")
|
| 72 |
SIDES = ("right", "back", "left")
|
| 73 |
QCODE_PROPERTY = "simready_metadata_qcode"
|
| 74 |
|
|
|
|
| 133 |
return views
|
| 134 |
|
| 135 |
|
| 136 |
+
# Name parts that say nothing about the object: Isaac's static-mesh prefix, Vertiv's root layer, the packaging.
|
| 137 |
+
PACKAGING = {"sm", "main", "simready", "usd"}
|
| 138 |
+
UUID = re.compile(r"[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}", re.I)
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def is_code(token: str, first: bool = False) -> bool:
|
| 142 |
+
"""An id, not a word: anything with a digit (D00272, f01, a3f9c2e1b7), one Latin letter, two Latin letters
|
| 143 |
+
unless they lead the name (tv_stand keeps tv; crx10ia_lp's lp is a suffix), or 8+ hex letters."""
|
| 144 |
+
return (any(c.isdigit() for c in token) or re.fullmatch(r"[A-Za-z]", token) is not None
|
| 145 |
+
or (not first and re.fullmatch(r"[A-Za-z]{2}", token) is not None)
|
| 146 |
+
or re.fullmatch(r"[0-9a-fA-F]{8,}", token) is not None)
|
| 147 |
+
|
| 148 |
+
|
| 149 |
def asset_hint(address: str) -> str:
|
| 150 |
+
"""The file name and its two nearest folders β never the host, owner or revision; "" when the file name is only
|
| 151 |
+
an id or a hash (Palatial's D00272.usd, a UUID), with no word for the model to read. No dictionary: a word in any
|
| 152 |
+
language or script counts."""
|
| 153 |
+
path = hf_addressing.parse(address).path
|
| 154 |
+
stem = UUID.sub(" ", path.rsplit("/", 1)[-1].rsplit(".", 1)[0])
|
| 155 |
+
tokens = [t for t in re.split(r"[\W_]+", stem) if t and t.lower() not in PACKAGING]
|
| 156 |
+
if all(is_code(t, first=i == 0) for i, t in enumerate(tokens)):
|
| 157 |
+
return ""
|
| 158 |
+
return "/".join(path.split("/")[-3:])
|
| 159 |
|
| 160 |
|
| 161 |
def media_tree(cfg: Config) -> Optional[Path]:
|
|
|
|
| 211 |
return not sha or (path.is_file() and hashlib.sha256(path.read_bytes()).hexdigest() == sha)
|
| 212 |
|
| 213 |
|
| 214 |
+
def mode_of(cfg: Config) -> str:
|
| 215 |
+
return cfg.get("VISION_VIEWS", "off").strip().lower() or "off"
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def is_on(cfg: Optional[Config]) -> bool:
|
| 219 |
+
"""Whether `index` runs this stage (VISION_VIEWS new or all): `export` then waits for it."""
|
| 220 |
+
return cfg is not None and mode_of(cfg) in ("new", "all")
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
class CaptionsFileError(ValueError):
|
| 224 |
+
"""VISION_VIEWS_APPLY's file does not match its .sha256 sibling."""
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def captions_file(value: str) -> Path:
|
| 228 |
+
"""VISION_VIEWS_APPLY's path: as given, else from the repo root (the env file names it from there)."""
|
| 229 |
+
path = Path(value)
|
| 230 |
+
return REPO_ROOT / path if not path.is_absolute() and not path.exists() else path
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def sha256_sibling(path: Path) -> Path:
|
| 234 |
+
"""captions.jsonl.gz and captions.jsonl both name captions.jsonl.sha256 (`sha256sum` of the uncompressed file)."""
|
| 235 |
+
name = path.name[:-3] if path.name.endswith(".gz") else path.name
|
| 236 |
+
return path.with_name(name + ".sha256")
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def read_captions(path: Path) -> Tuple[str, str]:
|
| 240 |
+
"""(text, sha256) of a captions file: gzip by suffix, checked against its .sha256 sibling when there is one."""
|
| 241 |
+
data = path.read_bytes()
|
| 242 |
+
if path.name.endswith(".gz"):
|
| 243 |
+
data = gzip.decompress(data)
|
| 244 |
+
got = hashlib.sha256(data).hexdigest()
|
| 245 |
+
sibling = sha256_sibling(path)
|
| 246 |
+
if sibling.is_file():
|
| 247 |
+
want = (sibling.read_text().split() or [""])[0].lower()
|
| 248 |
+
if got != want:
|
| 249 |
+
raise CaptionsFileError(f"{path.name}: sha256 {got} does not match {sibling.name} ({want or 'empty'})")
|
| 250 |
+
return data.decode("utf-8"), got
|
| 251 |
+
|
| 252 |
+
|
| 253 |
def apply_plan(cfg: Config, mode: str, path: Path) -> Dict:
|
| 254 |
"""Precomputed metadata to write as it is (no model call): {address: the writer's plugin fields}, for each
|
| 255 |
selected asset the file has at the SAME pinned address (same bytes) with a writer record. Others keep what the
|
| 256 |
thumbnail plugin wrote. An image-override asset is written only from a record made from its override picture;
|
| 257 |
+
else it keeps the image-overrides stage's caption. Raises CaptionsFileError when the file fails its sha256."""
|
| 258 |
from .image_overrides import load_spec
|
| 259 |
+
text, sha = read_captions(path)
|
| 260 |
records = {}
|
| 261 |
+
for line in text.splitlines():
|
| 262 |
if line.strip():
|
| 263 |
rec = json.loads(line)
|
| 264 |
if (rec.get("writer") or {}).get("plugin_fields"):
|
| 265 |
records[rec["address"]] = rec
|
| 266 |
overrides = load_spec()
|
| 267 |
+
assets, missing, wrong_image = {}, [], []
|
| 268 |
for address in select_addresses(cfg, mode, keep_curated=True):
|
| 269 |
rec = records.get(address)
|
| 270 |
if rec is None:
|
| 271 |
+
missing.append(address)
|
| 272 |
continue
|
| 273 |
if address in overrides and not override_image_matches(rec, overrides[address]):
|
| 274 |
+
wrong_image.append(address)
|
| 275 |
continue
|
| 276 |
assets[address] = {"apply": rec["writer"]["plugin_fields"], "writer_model": rec["writer"].get("model"),
|
| 277 |
"check_object_type": ((rec.get("check") or {}).get("meta") or {}).get("object_type"),
|
| 278 |
"models_agree": rec.get("agree")}
|
| 279 |
+
return {"index": cfg.main_index, "mode": "apply", "source": str(path), "sha256": sha, "media": None,
|
| 280 |
+
"assets": assets, "not_precomputed": len(missing), "override_image_mismatch": len(wrong_image),
|
| 281 |
+
"not_precomputed_assets": missing, "override_image_mismatch_assets": wrong_image}
|
| 282 |
|
| 283 |
|
| 284 |
def plan(cfg: Config, mode: str) -> Dict:
|
| 285 |
addresses = select_addresses(cfg, mode)
|
| 286 |
+
hint_on = cfg.get("VISION_VIEWS_HINT", "on").strip().lower() not in OFF
|
| 287 |
# thumbnail: every asset from its shipped thumbnail + the hint (measured as good as the views on object_type,
|
| 288 |
# at a quarter of the image tokens); views: the RTX views where the asset has them
|
| 289 |
views_on = cfg.get("VISION_VIEWS_IMAGES", "views").strip().lower() != "thumbnail"
|
|
|
|
| 380 |
|
| 381 |
|
| 382 |
def run(cfg: Config, dry_run: bool = False) -> int:
|
| 383 |
+
mode = mode_of(cfg)
|
| 384 |
if mode not in MODES:
|
| 385 |
print(f"VISION_VIEWS={mode!r}: expected one of {', '.join(MODES)}")
|
| 386 |
return 1
|
| 387 |
if mode == "off":
|
| 388 |
print("vision views: off (VISION_VIEWS) β captions stay as the thumbnail plugin wrote them")
|
| 389 |
return 0
|
| 390 |
+
apply_value = cfg.get("VISION_VIEWS_APPLY").strip()
|
| 391 |
+
if apply_value:
|
| 392 |
+
# write precomputed metadata (config/vision_captions/*.jsonl.gz): no model call, no API key, any usdsearch
|
| 393 |
+
# image (the worker's "usdsearch image too old" check guards generating captions only)
|
| 394 |
+
path = captions_file(apply_value)
|
| 395 |
+
if not path.is_file():
|
| 396 |
+
print(f"VISION_VIEWS_APPLY={apply_value!r}: no such file")
|
| 397 |
return 1
|
| 398 |
+
try:
|
| 399 |
+
spec = apply_plan(cfg, mode, path)
|
| 400 |
+
except CaptionsFileError as exc:
|
| 401 |
+
print(f"VISION_VIEWS_APPLY: {exc}; nothing written")
|
| 402 |
+
return 1
|
| 403 |
+
missing, wrong = spec.pop("not_precomputed_assets"), spec.pop("override_image_mismatch_assets")
|
| 404 |
+
listing = BUILD_DIR / APPLY_LISTING
|
| 405 |
+
shown = listing.relative_to(REPO_ROOT) if listing.is_relative_to(REPO_ROOT) else listing
|
| 406 |
+
if not dry_run:
|
| 407 |
+
record = {"source": apply_value, "sha256": spec["sha256"], "applied": len(spec["assets"]),
|
| 408 |
+
"not_in_file": missing, "override_image_mismatch": wrong}
|
| 409 |
+
listing.write_text(json.dumps(record, indent=1) + "\n")
|
| 410 |
+
print(f"vision views ({mode}, apply {apply_value}, sha256 {spec['sha256'][:12]}): {len(spec['assets'])} "
|
| 411 |
+
f"asset(s) get precomputed metadata; {len(missing)} not in the file keep their caption; {len(wrong)} "
|
| 412 |
+
f"image-override asset(s) whose record is not from their override image keep the image-overrides "
|
| 413 |
+
f"stage's; no VLM call; listed in {shown}")
|
| 414 |
+
for address in missing[:5]:
|
| 415 |
+
print(f" not in the file: {address}")
|
| 416 |
+
if len(missing) > 5:
|
| 417 |
+
print(f" ... and {len(missing) - 5} more")
|
| 418 |
if dry_run:
|
| 419 |
print("dry run: nothing written")
|
| 420 |
return 0
|
| 421 |
+
if not spec["assets"]:
|
| 422 |
+
return 0
|
| 423 |
return run_worker(cfg, spec)
|
| 424 |
if not dry_run and not os.environ.get("USDSEARCH_LLM_API_KEY", "").strip():
|
| 425 |
print("VISION_VIEWS is on but USDSEARCH_LLM_API_KEY is not set in the environment")
|
| 426 |
return 1
|
| 427 |
check_model = cfg.get("VISION_VIEWS_CHECK_MODEL").strip()
|
| 428 |
+
check_model = "" if check_model.lower() in OFF else check_model
|
| 429 |
on_disagree = cfg.get("VISION_VIEWS_ON_DISAGREE", "flag").strip().lower() or "flag"
|
| 430 |
if on_disagree not in ON_DISAGREE:
|
| 431 |
print(f"VISION_VIEWS_ON_DISAGREE={on_disagree!r}: expected one of {', '.join(ON_DISAGREE)}")
|
|
|
|
| 434 |
kinds = Counter(len(a["views"]) for a in spec["assets"].values())
|
| 435 |
opens = sum(1 for a in spec["assets"].values() if any(v["label"] == "open" for v in a["views"]))
|
| 436 |
images = sum(max(1, len(a["views"])) for a in spec["assets"].values())
|
| 437 |
+
hints = sum(1 for a in spec["assets"].values() if a["hint"])
|
| 438 |
calls = len(spec["assets"]) * (2 if check_model else 1)
|
| 439 |
print(f"vision views ({mode}): {len(spec['assets'])} asset(s), media {spec['media']}; views per asset "
|
| 440 |
+
f"{dict(sorted(kinds.items()))} (0 = thumbnail); {opens} with an open view; {images} image(s); {hints} with "
|
| 441 |
+
f"a name hint; {calls} VLM call(s)"
|
| 442 |
+
+ (f" (check model {check_model}, on disagreement: {on_disagree})" if check_model else ""))
|
| 443 |
if dry_run:
|
| 444 |
print("dry run: nothing written")
|
| 445 |
return 0
|
|
|
|
| 469 |
"-e", "USDSEARCH_VISION_METADATA_YAML_FILEPATH=/app/vlm_metadata_fields.yaml",
|
| 470 |
"-v", f"{REPO_ROOT / 'config' / 'vlm_metadata_prompt.txt'}:/app/vlm_metadata_prompt.txt:ro",
|
| 471 |
"-e", "USDSEARCH_VISION_METADATA_PROMPT_FILEPATH=/app/vlm_metadata_prompt.txt",
|
| 472 |
+
"-e", "OPENSEARCH_URL_INTERNAL=http://opensearch:9200",
|
| 473 |
+
# only the spec's hint (none for an id or a hash): never the plugin's own from omni_path
|
| 474 |
+
"-e", "VLM_ASSET_HINT=false"]
|
| 475 |
if spec["media"]:
|
| 476 |
cmd += ["-v", f"{spec['media']}:/media:ro"]
|
| 477 |
for var in passthrough:
|
pipeline/vision_views_worker.py
CHANGED
|
@@ -18,6 +18,7 @@ from image_overrides_worker import fetch_thumbnail, os_doc, result_content # no
|
|
| 18 |
from vision_views_rules import agrees, compatible # noqa: E402
|
| 19 |
|
| 20 |
CHECK_MODEL = (os.environ.get("VISION_VIEWS_CHECK_MODEL") or "").strip()
|
|
|
|
| 21 |
CHECK_EFFORT = (os.environ.get("VISION_VIEWS_CHECK_REASONING_EFFORT") or "").strip()
|
| 22 |
ON_DISAGREE = (os.environ.get("VISION_VIEWS_ON_DISAGREE") or "flag").strip().lower()
|
| 23 |
CONCURRENCY = int(os.environ.get("VISION_VIEWS_CONCURRENCY") or 4)
|
|
|
|
| 18 |
from vision_views_rules import agrees, compatible # noqa: E402
|
| 19 |
|
| 20 |
CHECK_MODEL = (os.environ.get("VISION_VIEWS_CHECK_MODEL") or "").strip()
|
| 21 |
+
CHECK_MODEL = "" if CHECK_MODEL.lower() in ("off", "none", "false", "0", "no") else CHECK_MODEL
|
| 22 |
CHECK_EFFORT = (os.environ.get("VISION_VIEWS_CHECK_REASONING_EFFORT") or "").strip()
|
| 23 |
ON_DISAGREE = (os.environ.get("VISION_VIEWS_ON_DISAGREE") or "flag").strip().lower()
|
| 24 |
CONCURRENCY = int(os.environ.get("VISION_VIEWS_CONCURRENCY") or 4)
|
services/explorer/dist/build-info.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
{
|
| 2 |
"product": "simready-central-slim-explorer",
|
| 3 |
-
"version": "slim-ui-
|
| 4 |
-
"bundle_id": "main.
|
| 5 |
-
"built_at": "2026-10-
|
| 6 |
-
"flags": {"public_mode": true, "email_auth": false, "create_ui": false, "hierarchy_editor": false, "create_search": false, "auth_ui": false, "credentials_ui": false, "index_management": false, "inference_backend": "hub", "vlm_asset_refs": false, "validation_image_max_px": 448, "validation_max_retries": 0, "validation_max_concurrent": 32, "validation_tail_concurrent": 8, "validation_eager_count": 50, "validation_batch_stream": true, "library_idle_prefetch": false, "validation_score_scale": "percent", "parse_gate_fields": true, "parse_fast_path": true, "featured_landing": true, "hf_download_commands": true, "capacity_level_url": "https://huggingface.co/datasets/nvidia/simready-central-status/resolve/main/status.json", "dependency_views": true, "model_viewer": false, "library_viewer": true, "library_media": "runtime", "spell_correction": true, "downloads": "runtime", "submit_content": "runtime", "third_party_notices": "runtime"}
|
| 7 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"product": "simready-central-slim-explorer",
|
| 3 |
+
"version": "slim-ui-f57ff00c",
|
| 4 |
+
"bundle_id": "main.e1d091ef.js",
|
| 5 |
+
"built_at": "2026-10-20T14:33:18Z",
|
| 6 |
+
"flags": {"public_mode": true, "email_auth": false, "create_ui": false, "hierarchy_editor": false, "create_search": false, "auth_ui": false, "credentials_ui": false, "index_management": false, "inference_backend": "hub", "vlm_asset_refs": false, "validation_image_max_px": 448, "validation_max_retries": 0, "validation_max_concurrent": 32, "validation_tail_concurrent": 8, "validation_eager_count": 50, "validation_batch_stream": true, "library_idle_prefetch": false, "validation_score_scale": "percent", "parse_gate_fields": true, "parse_fast_path": true, "featured_landing": true, "hf_download_commands": true, "capacity_level_url": "https://huggingface.co/datasets/nvidia/simready-central-status/resolve/main/status.json", "max_usd_mb": "", "notice_usd_mb": "50", "dependency_views": true, "model_viewer": false, "library_viewer": true, "library_media": "runtime", "spell_correction": true, "downloads": "runtime", "submit_content": "runtime", "third_party_notices": "runtime"}
|
| 7 |
}
|
services/explorer/dist/index.html
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/ds.png"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="SimReady Central β SimReady asset search"/><meta name="ui-version" content="slim-ui-
|
|
|
|
| 1 |
+
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/ds.png"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="SimReady Central β SimReady asset search"/><meta name="ui-version" content="slim-ui-f57ff00c"><script src="/config.js"></script><title>SimReady Central</title><script defer="defer" src="/static/js/main.e1d091ef.js"></script><link href="/static/css/main.ed151f6a.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
services/explorer/dist/static/css/{main.c80b163d.css β main.ed151f6a.css}
RENAMED
|
The diff for this file is too large to render.
See raw diff
|
|
|
services/explorer/dist/static/js/main.2f1a816a.js
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|
services/explorer/dist/static/js/main.e1d091ef.js
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
services/explorer/dist/static/js/{main.2f1a816a.js.LICENSE.txt β main.e1d091ef.js.LICENSE.txt}
RENAMED
|
File without changes
|
services/explorer/dist/third-party-notices.txt
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
Third-party notices β SimReady Central Explorer slim-ui-
|
| 2 |
|
| 3 |
The software in this web application's browser bundle includes the packages below, each under its own
|
| 4 |
license. Each entry gives the package, its version, its license and the license / notice text it ships.
|
|
|
|
| 1 |
+
Third-party notices β SimReady Central Explorer slim-ui-f57ff00c
|
| 2 |
|
| 3 |
The software in this web application's browser bundle includes the packages below, each under its own
|
| 4 |
license. Each entry gives the package, its version, its license and the license / notice text it ships.
|