Download 14B/code/video_fd/encoders/dinov3_adapter.py from Elastic-forcing/Elastic-forcing-code: direct link, hf CLI and curl.
- Browser
- Download file 35.6 kB
-
https://huggingface.co/Elastic-forcing/Elastic-forcing-code/resolve/main/14B/code/video_fd/encoders/dinov3_adapter.py
- Command line
-
hf download hf://Elastic-forcing/Elastic-forcing-code/14B/code/video_fd/encoders/dinov3_adapter.py
-
curl -L -o dinov3_adapter.py https://huggingface.co/Elastic-forcing/Elastic-forcing-code/resolve/main/14B/code/video_fd/encoders/dinov3_adapter.py
35.6 kB
| """Frozen Meta-native DINOv3 ViT-L/16 dense-video adapter. | |
| This adapter implements the one feature geometry used by the DINOv3 variant | |
| of the whole-video MMD objective: | |
| * input ``video_clip_01`` is ``[B, 81, 3, H, W]`` in ``[0, 1]``; | |
| * sixteen fixed frames are selected in chronological order; | |
| * each frame is directly resized to ``256 x 256`` and ImageNet-normalized; | |
| * Meta native ``forward_features()["x_norm_patchtokens"]`` supplies the final | |
| normalized ``16 x 16`` patch grid; | |
| * eight fixed rows crossed with eight fixed columns retain 64 patches/frame; | |
| * frame-major concatenation returns one ``[1024, 1024]`` matrix per video. | |
| The encoder always stays in eval mode with FP32 parameters and floating-point | |
| buffers. The first pass is detached, while :meth:`replay_dense_vjp` replays | |
| four selected frames at a time and returns a full 81-frame pixel cotangent | |
| without ever building a sixteen-frame encoder graph. | |
| """ | |
| from __future__ import annotations | |
| import importlib | |
| import subprocess | |
| import sys | |
| from collections.abc import Mapping, Sequence | |
| from contextlib import nullcontext | |
| from pathlib import Path | |
| from typing import Optional | |
| import torch | |
| from torch import nn | |
| import torch.nn.functional as F | |
| from video_fd.encoders.dinov3_dense_metadata import ( | |
| DEFAULT_WAN_VAE_ID, | |
| FEATURE_DIM, | |
| FORWARD_CHUNK_FRAMES, | |
| FRAME_INDICES, | |
| INPUT_SIZE, | |
| MODEL_ID, | |
| PATCH_GRID_SIZE, | |
| PATCH_SIZE, | |
| SOURCE_NUM_FRAMES, | |
| SOURCE_COMMIT, | |
| SPATIAL_AXIS_INDICES, | |
| canonical_dinov3_dense_feature_metadata, | |
| dense_token_indices, | |
| file_sha256, | |
| module_state_sha256, | |
| spatial_patch_indices, | |
| ) | |
| _IMAGENET_MEAN = (0.485, 0.456, 0.406) | |
| _IMAGENET_STD = (0.229, 0.224, 0.225) | |
| _NATIVE_VITL16_PARAMETER_COUNT = 303_154_176 | |
| _NATIVE_BACKBONES_MODULE = "dinov3.hub.backbones" | |
| def _dinov3_module_names() -> tuple[str, ...]: | |
| """Return a stable snapshot of every cached module in the DINOv3 tree.""" | |
| return tuple( | |
| name | |
| for name in tuple(sys.modules) | |
| if name == "dinov3" or name.startswith("dinov3.") | |
| ) | |
| def _assert_native_module_origin( | |
| module_name: str, | |
| module: object, | |
| repo: Path, | |
| ) -> Path: | |
| """Require a cached native module to be the matching file in ``repo``. | |
| Merely prepending ``repo`` to ``sys.path`` is insufficient because Python | |
| consults ``sys.modules`` first. Validate both normal modules and package | |
| search paths so an earlier import cannot redirect a relative import to a | |
| different DINOv3 checkout or an installed package. | |
| """ | |
| if module is None: | |
| raise RuntimeError( | |
| f"cached module {module_name!r} is incomplete; refusing DINOv3 import" | |
| ) | |
| origin = getattr(module, "__file__", None) | |
| if not isinstance(origin, str) or not origin: | |
| raise RuntimeError( | |
| f"cached module {module_name!r} has no filesystem origin; " | |
| "refusing DINOv3 import" | |
| ) | |
| try: | |
| resolved_origin = Path(origin).expanduser().resolve(strict=True) | |
| except (OSError, RuntimeError) as exc: | |
| raise RuntimeError( | |
| f"cannot resolve cached module {module_name!r} origin {origin!r}" | |
| ) from exc | |
| relative_parts = module_name.split(".") | |
| module_base = repo.joinpath(*relative_parts) | |
| expected_origins = { | |
| module_base.with_suffix(".py").resolve(), | |
| (module_base / "__init__.py").resolve(), | |
| } | |
| if resolved_origin not in expected_origins: | |
| raise RuntimeError( | |
| f"cached module {module_name!r} comes from {resolved_origin}, " | |
| f"not the pinned DINOv3 repo {repo}" | |
| ) | |
| package_paths = getattr(module, "__path__", None) | |
| if package_paths is not None: | |
| try: | |
| resolved_paths = tuple( | |
| Path(path).expanduser().resolve(strict=True) | |
| for path in package_paths | |
| ) | |
| except (OSError, RuntimeError, TypeError) as exc: | |
| raise RuntimeError( | |
| f"cached package {module_name!r} has invalid search paths" | |
| ) from exc | |
| expected_package_path = module_base.resolve(strict=True) | |
| if resolved_paths != (expected_package_path,): | |
| raise RuntimeError( | |
| f"cached package {module_name!r} search paths {resolved_paths!r} " | |
| f"do not equal the pinned path {(expected_package_path,)!r}" | |
| ) | |
| return resolved_origin | |
| def _validate_cached_dinov3_modules(repo: Path) -> None: | |
| for module_name in _dinov3_module_names(): | |
| _assert_native_module_origin( | |
| module_name, | |
| sys.modules.get(module_name), | |
| repo, | |
| ) | |
| def _remove_new_dinov3_modules(previous_names: set[str]) -> None: | |
| for module_name in _dinov3_module_names(): | |
| if module_name not in previous_names: | |
| sys.modules.pop(module_name, None) | |
| def _import_native_vitl16_factory(repo: Path): | |
| """Import only the official backbone module from one verified checkout. | |
| The repository's ``hubconf.py`` imports optional segmentation, detection, | |
| depth, and text dependencies even though this adapter needs only a vision | |
| backbone. Directly importing ``dinov3.hub.backbones`` avoids those optional | |
| dependency edges while preserving Meta's native architecture factory. | |
| """ | |
| repo = repo.expanduser().resolve(strict=True) | |
| if not (repo / "dinov3" / "hub" / "backbones.py").is_file(): | |
| raise FileNotFoundError( | |
| f"pinned DINOv3 backbone module is missing under {repo}" | |
| ) | |
| _validate_cached_dinov3_modules(repo) | |
| previous_module_names = set(_dinov3_module_names()) | |
| previous_sys_path = list(sys.path) | |
| try: | |
| sys.path.insert(0, str(repo)) | |
| importlib.invalidate_caches() | |
| module = importlib.import_module(_NATIVE_BACKBONES_MODULE) | |
| _assert_native_module_origin(_NATIVE_BACKBONES_MODULE, module, repo) | |
| _validate_cached_dinov3_modules(repo) | |
| factory = getattr(module, MODEL_ID, None) | |
| if not callable(factory): | |
| raise RuntimeError( | |
| f"{_NATIVE_BACKBONES_MODULE} does not expose callable {MODEL_ID}" | |
| ) | |
| return factory | |
| except Exception: | |
| _remove_new_dinov3_modules(previous_module_names) | |
| raise | |
| finally: | |
| sys.path[:] = previous_sys_path | |
| def _build_native_vitl16(repo: Path) -> nn.Module: | |
| """Construct the exact official ViT-L/16 architecture without weights.""" | |
| factory = _import_native_vitl16_factory(repo) | |
| previous_module_names = set(_dinov3_module_names()) | |
| previous_sys_path = list(sys.path) | |
| try: | |
| model = factory(pretrained=False) | |
| _validate_cached_dinov3_modules(repo) | |
| if not isinstance(model, nn.Module): | |
| raise TypeError( | |
| f"official {MODEL_ID} factory did not return an nn.Module" | |
| ) | |
| return model | |
| except Exception: | |
| _remove_new_dinov3_modules(previous_module_names) | |
| raise | |
| finally: | |
| # Some optional acceleration imports used by the native constructor | |
| # append transient paths. Keep the adapter's source-loading boundary | |
| # side-effect free regardless of whether construction succeeds. | |
| sys.path[:] = previous_sys_path | |
| def _autocast_disabled(device: torch.device): | |
| if device.type in {"cpu", "cuda"}: | |
| return torch.autocast(device_type=device.type, enabled=False) | |
| return nullcontext() | |
| def _strict_config_tuple( | |
| config, | |
| key: str, | |
| expected: tuple[int, ...], | |
| ) -> tuple[int, ...]: | |
| raw = config.get(key, expected) | |
| if isinstance(raw, (str, bytes)): | |
| raise TypeError(f"{key} must be a sequence of integers") | |
| values = tuple(raw) | |
| if any(type(value) is not int for value in values): | |
| raise TypeError(f"{key} must contain Python integers") | |
| if values != expected: | |
| raise ValueError(f"{key} must equal {list(expected)}") | |
| return values | |
| def _is_tensor_state_dict(value) -> bool: | |
| return ( | |
| isinstance(value, Mapping) | |
| and len(value) > 0 | |
| and all(isinstance(key, str) for key in value) | |
| and all(torch.is_tensor(tensor) for tensor in value.values()) | |
| ) | |
| def _checkpoint_state_dict(payload) -> Mapping[str, torch.Tensor]: | |
| """Accept only a direct state dict or an explicit ``model`` wrapper.""" | |
| if _is_tensor_state_dict(payload): | |
| return payload | |
| if isinstance(payload, Mapping) and "model" in payload: | |
| state = payload["model"] | |
| if _is_tensor_state_dict(state): | |
| return state | |
| raise TypeError("checkpoint['model'] is not a non-empty tensor state dict") | |
| raise TypeError( | |
| "DINOv3 checkpoint must be a direct tensor state dict or contain " | |
| "one under the explicit 'model' key" | |
| ) | |
| class DINOv3DenseAdapter(nn.Module): | |
| """Meta-native DINOv3 ViT-L/16 adapter with an exact replay VJP API. | |
| Required production config keys are ``repo_path``, ``weight_path`` and | |
| ``source_commit``. Geometry keys may be omitted because their canonical | |
| values are defaults, but supplying a different value is rejected. Passing | |
| ``model=...`` is a unit-test seam; production construction creates the | |
| official architecture with ``pretrained=False`` and then strict-loads the | |
| local checkpoint itself. | |
| """ | |
| def __init__(self, config, model: Optional[nn.Module] = None): | |
| super().__init__() | |
| config = dict(config) | |
| self.config = config | |
| self.model_id = str(config.get("model_id", MODEL_ID)) | |
| if self.model_id != MODEL_ID: | |
| raise ValueError(f"model_id must be {MODEL_ID!r}") | |
| self.repo_path = str(config.get("repo_path", "")) | |
| self.weight_path = str(config.get("weight_path", "")) | |
| self.source_commit = str(config.get("source_commit", "")) | |
| for name, value in ( | |
| ("repo_path", self.repo_path), | |
| ("weight_path", self.weight_path), | |
| ): | |
| if not value or "\0" in value: | |
| raise ValueError(f"{name} must be a non-empty string without NUL characters") | |
| if self.source_commit != SOURCE_COMMIT: | |
| raise ValueError( | |
| "source_commit must equal the pinned DINOv3 source commit " | |
| f"{SOURCE_COMMIT}" | |
| ) | |
| self.source_num_frames = int( | |
| config.get("source_num_frames", SOURCE_NUM_FRAMES)) | |
| if self.source_num_frames != SOURCE_NUM_FRAMES: | |
| raise ValueError(f"source_num_frames must be {SOURCE_NUM_FRAMES}") | |
| self.frame_indices = _strict_config_tuple( | |
| config, "frame_indices", FRAME_INDICES) | |
| self.dense_num_frames = len(self.frame_indices) | |
| self.input_size = int(config.get("input_size", INPUT_SIZE)) | |
| if self.input_size != INPUT_SIZE: | |
| raise ValueError(f"input_size must be {INPUT_SIZE}") | |
| self.patch_size = int(config.get("patch_size", PATCH_SIZE)) | |
| if self.patch_size != PATCH_SIZE: | |
| raise ValueError(f"patch_size must be {PATCH_SIZE}") | |
| configured_grid = int(config.get("patch_grid_size", PATCH_GRID_SIZE)) | |
| if configured_grid != PATCH_GRID_SIZE: | |
| raise ValueError(f"patch_grid_size must be {PATCH_GRID_SIZE}") | |
| self.patch_grid_size = configured_grid | |
| self.spatial_axis_indices = _strict_config_tuple( | |
| config, "spatial_axis_indices", SPATIAL_AXIS_INDICES) | |
| self.spatial_patch_indices = tuple( | |
| int(value) for value in spatial_patch_indices().tolist()) | |
| # ``spatial_token_indices`` is the name used by offline reference | |
| # tooling; both aliases describe the same per-frame 16x16 patch grid. | |
| self.spatial_token_indices = self.spatial_patch_indices | |
| supplied_spatial = config.get("spatial_token_indices") | |
| if supplied_spatial is not None: | |
| supplied = ( | |
| supplied_spatial.detach() | |
| if isinstance(supplied_spatial, torch.Tensor) | |
| else torch.tensor(tuple(supplied_spatial)) | |
| ) | |
| if ( | |
| supplied.dtype == torch.bool | |
| or supplied.is_floating_point() | |
| or supplied.is_complex() | |
| ): | |
| raise TypeError("spatial_token_indices must contain integers") | |
| supplied = supplied.detach().to(device="cpu", dtype=torch.long) | |
| if not torch.equal(supplied, spatial_patch_indices()): | |
| raise ValueError( | |
| "spatial_token_indices must equal the canonical 64 patch indices") | |
| self.forward_chunk_frames = int( | |
| config.get("forward_chunk_frames", FORWARD_CHUNK_FRAMES)) | |
| if self.forward_chunk_frames != FORWARD_CHUNK_FRAMES: | |
| raise ValueError( | |
| f"forward_chunk_frames must be {FORWARD_CHUNK_FRAMES}") | |
| self.dense_feature_dim = int(config.get("feature_dim", FEATURE_DIM)) | |
| if self.dense_feature_dim != FEATURE_DIM: | |
| raise ValueError(f"feature_dim must be {FEATURE_DIM}") | |
| self.per_frame_tokens = len(self.spatial_patch_indices) | |
| if int(config.get("per_frame_tokens", self.per_frame_tokens)) != self.per_frame_tokens: | |
| raise ValueError(f"per_frame_tokens must be {self.per_frame_tokens}") | |
| self.dense_raw_tokens = ( | |
| self.dense_num_frames * self.patch_grid_size * self.patch_grid_size) | |
| self.dense_num_tokens = self.dense_num_frames * self.per_frame_tokens | |
| if int(config.get("num_tokens", self.dense_num_tokens)) != self.dense_num_tokens: | |
| raise ValueError(f"num_tokens must be {self.dense_num_tokens}") | |
| self.vae_id = str(config.get("vae_id", DEFAULT_WAN_VAE_ID)) | |
| self.raw_tokens = self.dense_raw_tokens | |
| self.num_tokens = self.dense_num_tokens | |
| self.feature_dim = self.dense_feature_dim | |
| self.register_buffer( | |
| "_frame_indices_tensor", | |
| torch.tensor(self.frame_indices, dtype=torch.long), | |
| persistent=False, | |
| ) | |
| self.register_buffer( | |
| "_spatial_patch_indices_tensor", | |
| spatial_patch_indices(), | |
| persistent=False, | |
| ) | |
| self.register_buffer( | |
| "_norm_mean", | |
| torch.tensor(_IMAGENET_MEAN, dtype=torch.float32).view(1, 3, 1, 1), | |
| persistent=False, | |
| ) | |
| self.register_buffer( | |
| "_norm_std", | |
| torch.tensor(_IMAGENET_STD, dtype=torch.float32).view(1, 3, 1, 1), | |
| persistent=False, | |
| ) | |
| configured_weight_sha = config.get( | |
| "weight_sha256", config.get("checkpoint_sha256")) | |
| self._injected_model = model is not None | |
| if model is None: | |
| loaded_model = self._load_native_model() | |
| else: | |
| loaded_model = model | |
| self._adopt_model(loaded_model) | |
| if self._injected_model: | |
| actual_weight_sha = module_state_sha256(self._model) | |
| else: | |
| actual_weight_sha = file_sha256(self.weight_path) | |
| if configured_weight_sha is not None: | |
| configured_weight_sha = str(configured_weight_sha).lower() | |
| if configured_weight_sha != actual_weight_sha: | |
| raise ValueError( | |
| "configured weight_sha256 does not match the loaded model/checkpoint: " | |
| f"{configured_weight_sha} != {actual_weight_sha}" | |
| ) | |
| self.weight_sha256 = actual_weight_sha | |
| def model(self) -> nn.Module: | |
| return self._model | |
| def dense_token_indices(self) -> tuple[int, ...]: | |
| return tuple(int(value) for value in dense_token_indices().tolist()) | |
| def _load_native_model(self) -> nn.Module: | |
| repo = Path(self.repo_path).expanduser().resolve() | |
| weights = Path(self.weight_path).expanduser().resolve() | |
| if not repo.is_dir(): | |
| raise FileNotFoundError(f"DINOv3 repo_path is not a directory: {repo}") | |
| if not weights.is_file(): | |
| raise FileNotFoundError(f"DINOv3 weight_path is not a file: {weights}") | |
| self._verify_repo_source(repo) | |
| # The official vitl16 helper parses an 8-hex suffix whenever a custom | |
| # weights string is passed. Build without pretrained weights, then load | |
| # the user's pinned local file independently so arbitrary safe filenames | |
| # work without renaming or symlinking. | |
| model = _build_native_vitl16(repo) | |
| payload = torch.load( | |
| str(weights), | |
| map_location="cpu", | |
| weights_only=True, | |
| ) | |
| state_dict = _checkpoint_state_dict(payload) | |
| model.load_state_dict(state_dict, strict=True) | |
| return model | |
| def _verify_repo_source(repo: Path) -> None: | |
| """Require the pinned Git commit and no tracked or untracked changes.""" | |
| def run_git(*arguments: str) -> str: | |
| command = ["git", "-C", str(repo), *arguments] | |
| try: | |
| result = subprocess.run( | |
| command, | |
| check=True, | |
| capture_output=True, | |
| text=True, | |
| ) | |
| except (OSError, subprocess.CalledProcessError) as exc: | |
| raise RuntimeError( | |
| "failed to verify the pinned DINOv3 source repository" | |
| ) from exc | |
| return result.stdout | |
| actual_commit = run_git("rev-parse", "--verify", "HEAD").strip() | |
| if actual_commit != SOURCE_COMMIT: | |
| raise RuntimeError( | |
| "DINOv3 repo HEAD does not match the pinned source commit: " | |
| f"{actual_commit!r} != {SOURCE_COMMIT!r}" | |
| ) | |
| tracked_status = run_git( | |
| "status", | |
| "--porcelain", | |
| "--untracked-files=all", | |
| ) | |
| if tracked_status.strip(): | |
| raise RuntimeError( | |
| "DINOv3 repo has worktree changes; refusing to load " | |
| "an unpinned native implementation" | |
| ) | |
| def _adopt_model(self, model: nn.Module) -> None: | |
| if not isinstance(model, nn.Module): | |
| raise TypeError("model injection must be an nn.Module") | |
| self._model = model.to(dtype=torch.float32) | |
| self._model.eval().requires_grad_(False) | |
| self._validate_native_geometry() | |
| self._verify_model_fp32() | |
| # Use nn.Module.train directly so our override cannot accidentally | |
| # propagate ``training=True`` into the positional-coordinate path. | |
| nn.Module.train(self, False) | |
| def _validate_native_geometry(self) -> None: | |
| if not callable(getattr(self._model, "forward_features", None)): | |
| raise TypeError("DINOv3 model must expose callable forward_features") | |
| for attribute, expected in ( | |
| ("patch_size", PATCH_SIZE), | |
| ("embed_dim", FEATURE_DIM), | |
| ("n_storage_tokens", 4), | |
| ): | |
| if not hasattr(self._model, attribute): | |
| raise AttributeError( | |
| f"DINOv3 model is missing required attribute {attribute!r}") | |
| actual = getattr(self._model, attribute) | |
| if isinstance(actual, (tuple, list)): | |
| valid = len(actual) > 0 and all(int(value) == expected for value in actual) | |
| else: | |
| valid = int(actual) == expected | |
| if not valid: | |
| raise ValueError( | |
| f"DINOv3 {attribute} mismatch: expected {expected}, got {actual}") | |
| blocks = getattr(self._model, "blocks", None) | |
| if blocks is None or len(blocks) != 24: | |
| raise ValueError( | |
| "DINOv3 ViT-L/16 must expose exactly 24 transformer blocks") | |
| if not self._injected_model: | |
| parameter_count = sum(parameter.numel() for parameter in self._model.parameters()) | |
| if parameter_count != _NATIVE_VITL16_PARAMETER_COUNT: | |
| raise ValueError( | |
| "DINOv3 ViT-L/16 parameter count mismatch: " | |
| f"{parameter_count} != {_NATIVE_VITL16_PARAMETER_COUNT}" | |
| ) | |
| def _verify_model_fp32(self) -> None: | |
| wrong = [ | |
| (f"parameter:{name}", value.dtype) | |
| for name, value in self._model.named_parameters() | |
| if value.dtype != torch.float32 | |
| ] | |
| wrong.extend( | |
| (f"buffer:{name}", value.dtype) | |
| for name, value in self._model.named_buffers() | |
| if value.is_floating_point() and value.dtype != torch.float32 | |
| ) | |
| if wrong: | |
| preview = ", ".join(f"{name}={dtype}" for name, dtype in wrong[:8]) | |
| raise TypeError( | |
| "DINOv3 parameters and floating-point buffers must be FP32; " | |
| f"found {preview}" | |
| ) | |
| if any(parameter.requires_grad for parameter in self._model.parameters()): | |
| raise RuntimeError("DINOv3 parameters must be frozen") | |
| if self._model.training: | |
| raise RuntimeError("DINOv3 model must stay in eval mode") | |
| def train(self, mode: bool = True): | |
| """The frozen feature identity never permits training-mode augmentations.""" | |
| nn.Module.train(self, False) | |
| self._model.eval() | |
| return self | |
| def _validate_video(self, video_clip_01: torch.Tensor) -> None: | |
| if not isinstance(video_clip_01, torch.Tensor): | |
| raise TypeError("video_clip_01 must be a torch.Tensor") | |
| if ( | |
| video_clip_01.ndim != 5 | |
| or int(video_clip_01.shape[0]) <= 0 | |
| or int(video_clip_01.shape[1]) != self.source_num_frames | |
| or int(video_clip_01.shape[2]) != 3 | |
| or int(video_clip_01.shape[3]) <= 0 | |
| or int(video_clip_01.shape[4]) <= 0 | |
| ): | |
| raise ValueError( | |
| "video_clip_01 must be a non-empty " | |
| f"[B,{self.source_num_frames},3,H,W] tensor, got " | |
| f"{tuple(video_clip_01.shape)}" | |
| ) | |
| if video_clip_01.dtype != torch.float32: | |
| raise TypeError( | |
| "video_clip_01 must be FP32, got " | |
| f"{video_clip_01.dtype}" | |
| ) | |
| if not bool(torch.isfinite(video_clip_01).all().item()): | |
| raise FloatingPointError("video_clip_01 contains non-finite values") | |
| minimum = float(video_clip_01.min().item()) | |
| maximum = float(video_clip_01.max().item()) | |
| if minimum < 0.0 or maximum > 1.0: | |
| raise ValueError( | |
| "video_clip_01 values must lie in [0, 1], got range " | |
| f"[{minimum:.8g}, {maximum:.8g}]" | |
| ) | |
| def _resolve_token_indices( | |
| self, | |
| token_indices: Optional[torch.Tensor | Sequence[int]], | |
| ) -> torch.Tensor: | |
| canonical = self._spatial_patch_indices_tensor | |
| if token_indices is None: | |
| return canonical | |
| supplied = ( | |
| token_indices.detach() | |
| if isinstance(token_indices, torch.Tensor) | |
| else torch.tensor(tuple(token_indices)) | |
| ) | |
| if ( | |
| supplied.dtype == torch.bool | |
| or supplied.is_floating_point() | |
| or supplied.is_complex() | |
| ): | |
| raise TypeError("token_indices must contain integers") | |
| supplied = supplied.detach().to( | |
| device=canonical.device, dtype=torch.long).contiguous() | |
| if not torch.equal(supplied, canonical): | |
| raise ValueError( | |
| "token_indices must equal the canonical 64 flattened spatial indices" | |
| ) | |
| return canonical | |
| def _selected_flat_frames(self, video_clip_01: torch.Tensor) -> torch.Tensor: | |
| selected = video_clip_01.index_select( | |
| 1, self._frame_indices_tensor.to(video_clip_01.device)) | |
| return selected.reshape(-1, *selected.shape[2:]).contiguous() | |
| def preprocess_frames(self, frames_01: torch.Tensor) -> torch.Tensor: | |
| """Direct square resize and ImageNet-normalize ``[N,3,H,W]`` frames.""" | |
| if ( | |
| not isinstance(frames_01, torch.Tensor) | |
| or frames_01.ndim != 4 | |
| or int(frames_01.shape[0]) <= 0 | |
| or int(frames_01.shape[1]) != 3 | |
| or not frames_01.is_floating_point() | |
| ): | |
| raise ValueError("frames_01 must be a non-empty floating [N,3,H,W] tensor") | |
| with _autocast_disabled(frames_01.device): | |
| frames = frames_01.float() | |
| if tuple(frames.shape[-2:]) != (self.input_size, self.input_size): | |
| frames = F.interpolate( | |
| frames, | |
| size=(self.input_size, self.input_size), | |
| mode="bilinear", | |
| align_corners=False, | |
| antialias=True, | |
| ) | |
| return ( | |
| frames - self._norm_mean.to(device=frames.device) | |
| ) / self._norm_std.to(device=frames.device) | |
| def preprocess_dense(self, video_clip_01: torch.Tensor) -> torch.Tensor: | |
| """Return all sixteen selected, normalized frames as ``[B,16,3,256,256]``.""" | |
| self._validate_video(video_clip_01) | |
| batch_size = int(video_clip_01.shape[0]) | |
| flat = self._selected_flat_frames(video_clip_01) | |
| processed = [] | |
| for start in range(0, int(flat.shape[0]), self.forward_chunk_frames): | |
| processed.append(self.preprocess_frames( | |
| flat[start:start + self.forward_chunk_frames])) | |
| return torch.cat(processed, dim=0).reshape( | |
| batch_size, | |
| self.dense_num_frames, | |
| 3, | |
| self.input_size, | |
| self.input_size, | |
| ) | |
| def _encode_frame_batch( | |
| self, | |
| frames_01: torch.Tensor, | |
| token_indices: torch.Tensor, | |
| ) -> torch.Tensor: | |
| pixel_values = self.preprocess_frames(frames_01) | |
| with _autocast_disabled(pixel_values.device): | |
| outputs = self._model.forward_features(pixel_values) | |
| if not isinstance(outputs, Mapping) or "x_norm_patchtokens" not in outputs: | |
| raise TypeError( | |
| "DINOv3 forward_features must return a mapping containing " | |
| "'x_norm_patchtokens'" | |
| ) | |
| patches = outputs["x_norm_patchtokens"] | |
| expected_shape = ( | |
| int(frames_01.shape[0]), | |
| self.patch_grid_size * self.patch_grid_size, | |
| self.dense_feature_dim, | |
| ) | |
| if not torch.is_tensor(patches) or tuple(patches.shape) != expected_shape: | |
| actual = None if not torch.is_tensor(patches) else tuple(patches.shape) | |
| raise RuntimeError( | |
| "unexpected DINOv3 normalized patch feature shape: " | |
| f"expected {expected_shape}, got {actual}" | |
| ) | |
| if patches.dtype != torch.float32: | |
| raise TypeError( | |
| f"DINOv3 normalized patch features must be FP32, got {patches.dtype}") | |
| if not bool(torch.isfinite(patches).all().item()): | |
| raise FloatingPointError( | |
| "DINOv3 x_norm_patchtokens contains non-finite values" | |
| ) | |
| return patches.index_select(1, token_indices.to(patches.device)) | |
| def forward_dense( | |
| self, | |
| video_clip_01: torch.Tensor, | |
| token_indices: Optional[torch.Tensor | Sequence[int]] = None, | |
| gradient_checkpointing: bool = False, | |
| ) -> torch.Tensor: | |
| """Extract one frame-major ``[1024,1024]`` matrix per 81-frame video. | |
| ``gradient_checkpointing`` is accepted for compatibility with the old | |
| V-JEPA2 call boundary. DINOv3's independent-frame graph is bounded by | |
| the mandatory four-frame chunks, so no additional checkpoint wrapper is | |
| needed and the flag does not change feature identity. | |
| """ | |
| if type(gradient_checkpointing) is not bool: | |
| raise TypeError("gradient_checkpointing must be a bool") | |
| self._verify_model_fp32() | |
| self._validate_video(video_clip_01) | |
| resolved_tokens = self._resolve_token_indices(token_indices) | |
| batch_size = int(video_clip_01.shape[0]) | |
| flat_frames = self._selected_flat_frames(video_clip_01) | |
| chunks = [] | |
| for start in range(0, int(flat_frames.shape[0]), self.forward_chunk_frames): | |
| chunks.append(self._encode_frame_batch( | |
| flat_frames[start:start + self.forward_chunk_frames], | |
| resolved_tokens, | |
| )) | |
| features = torch.cat(chunks, dim=0) | |
| return features.reshape( | |
| batch_size, | |
| self.dense_num_frames * self.per_frame_tokens, | |
| self.dense_feature_dim, | |
| ) | |
| def forward_dense_no_grad( | |
| self, | |
| video_clip_01: torch.Tensor, | |
| token_indices: Optional[torch.Tensor | Sequence[int]] = None, | |
| gradient_checkpointing: bool = False, | |
| ) -> torch.Tensor: | |
| """Detached memory-bounded first-pass extraction.""" | |
| return self.forward_dense( | |
| video_clip_01, | |
| token_indices=token_indices, | |
| gradient_checkpointing=gradient_checkpointing, | |
| ).detach().float().clone() | |
| extract_dense_no_grad = forward_dense_no_grad | |
| def replay_dense_vjp( | |
| self, | |
| video_clip_01: torch.Tensor, | |
| feature_cotangent: torch.Tensor, | |
| *, | |
| expected_features: Optional[torch.Tensor] = None, | |
| token_indices: Optional[torch.Tensor | Sequence[int]] = None, | |
| gradient_checkpointing: bool = False, | |
| replay_atol: float = 1.0e-6, | |
| replay_rtol: float = 1.0e-5, | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| """Replay the feature VJP in four-frame chunks. | |
| Returns ``(replay_features, pixel_cotangent)``. The cotangent is FP32 | |
| with the original ``[B,81,3,H,W]`` layout; all 65 unselected frames are | |
| exactly zero. If ``expected_features`` is supplied, the replayed | |
| features must match the detached first pass before any cotangent is | |
| returned. | |
| """ | |
| if type(gradient_checkpointing) is not bool: | |
| raise TypeError("gradient_checkpointing must be a bool") | |
| self._verify_model_fp32() | |
| self._validate_video(video_clip_01) | |
| batch_size = int(video_clip_01.shape[0]) | |
| expected_shape = ( | |
| batch_size, | |
| self.dense_num_tokens, | |
| self.dense_feature_dim, | |
| ) | |
| if ( | |
| not isinstance(feature_cotangent, torch.Tensor) | |
| or not feature_cotangent.is_floating_point() | |
| or tuple(feature_cotangent.shape) != expected_shape | |
| or not bool(torch.isfinite(feature_cotangent).all().item()) | |
| ): | |
| actual = ( | |
| tuple(feature_cotangent.shape) | |
| if isinstance(feature_cotangent, torch.Tensor) | |
| else type(feature_cotangent).__name__ | |
| ) | |
| raise ValueError( | |
| "feature_cotangent must be a finite floating tensor with " | |
| f"shape {expected_shape}, got {actual}" | |
| ) | |
| if replay_atol < 0 or replay_rtol < 0: | |
| raise ValueError("replay tolerances must be non-negative") | |
| resolved_tokens = self._resolve_token_indices(token_indices) | |
| flat_frames = self._selected_flat_frames(video_clip_01.detach()) | |
| flat_cotangent = feature_cotangent.detach().reshape( | |
| batch_size * self.dense_num_frames, | |
| self.per_frame_tokens, | |
| self.dense_feature_dim, | |
| ).float() | |
| replay_chunks = [] | |
| gradient_chunks = [] | |
| for start in range(0, int(flat_frames.shape[0]), self.forward_chunk_frames): | |
| end = start + self.forward_chunk_frames | |
| frame_leaf = ( | |
| flat_frames[start:end] | |
| .detach() | |
| .clone() | |
| .requires_grad_(True) | |
| ) | |
| chunk_features = self._encode_frame_batch( | |
| frame_leaf, | |
| resolved_tokens, | |
| ) | |
| chunk_cotangent = flat_cotangent[start:end].to( | |
| device=chunk_features.device, | |
| dtype=torch.float32, | |
| ) | |
| surrogate = (chunk_features * chunk_cotangent).sum() | |
| (chunk_gradient,) = torch.autograd.grad( | |
| surrogate, | |
| frame_leaf, | |
| retain_graph=False, | |
| create_graph=False, | |
| ) | |
| replay_chunks.append(chunk_features.detach().float()) | |
| gradient_chunks.append(chunk_gradient.detach().float()) | |
| replay_features = torch.cat(replay_chunks, dim=0).reshape(expected_shape) | |
| if expected_features is not None: | |
| if ( | |
| not isinstance(expected_features, torch.Tensor) | |
| or tuple(expected_features.shape) != expected_shape | |
| or not bool(torch.isfinite(expected_features).all().item()) | |
| ): | |
| raise ValueError( | |
| f"expected_features must be finite with shape {expected_shape}") | |
| comparison = expected_features.detach().to( | |
| device=replay_features.device, | |
| dtype=torch.float32, | |
| ) | |
| if not torch.allclose( | |
| replay_features, | |
| comparison, | |
| atol=float(replay_atol), | |
| rtol=float(replay_rtol), | |
| ): | |
| max_abs = float( | |
| (replay_features - comparison).abs().max().item()) | |
| raise RuntimeError( | |
| "DINOv3 replay features changed from the first pass: " | |
| f"max_abs={max_abs:.8g}, atol={replay_atol}, rtol={replay_rtol}" | |
| ) | |
| selected_gradient = torch.cat(gradient_chunks, dim=0).reshape( | |
| batch_size, | |
| self.dense_num_frames, | |
| *video_clip_01.shape[2:], | |
| ) | |
| full_pixel_cotangent = torch.zeros( | |
| video_clip_01.shape, | |
| device=video_clip_01.device, | |
| dtype=torch.float32, | |
| ) | |
| full_pixel_cotangent.index_copy_( | |
| 1, | |
| self._frame_indices_tensor.to(video_clip_01.device), | |
| selected_gradient, | |
| ) | |
| if not bool(torch.isfinite(full_pixel_cotangent).all().item()): | |
| raise FloatingPointError("DINOv3 replay returned a non-finite pixel cotangent") | |
| return replay_features, full_pixel_cotangent | |
| def feature_metadata(self) -> dict[str, object]: | |
| """Return the strict checkpoint, preprocessing and matrix identity.""" | |
| return canonical_dinov3_dense_feature_metadata( | |
| repo_path=self.repo_path, | |
| weight_path=self.weight_path, | |
| source_commit=self.source_commit, | |
| weight_sha256=self.weight_sha256, | |
| source_num_frames=self.source_num_frames, | |
| frame_indices=self.frame_indices, | |
| input_size=self.input_size, | |
| patch_size=self.patch_size, | |
| patch_grid_size=self.patch_grid_size, | |
| spatial_axis_indices=self.spatial_axis_indices, | |
| spatial_token_indices=self.spatial_patch_indices, | |
| per_frame_tokens=self.per_frame_tokens, | |
| num_tokens=self.dense_num_tokens, | |
| forward_chunk_frames=self.forward_chunk_frames, | |
| feature_dim=self.dense_feature_dim, | |
| model_id=self.model_id, | |
| vae_id=self.vae_id, | |
| ) | |
| # Short alias for callers that already name the encoder without the geometry | |
| # suffix. Both names refer to the same strict implementation. | |
| DINOv3Adapter = DINOv3DenseAdapter | |
| __all__ = ["DINOv3Adapter", "DINOv3DenseAdapter"] | |