Video Classification
Transformers
Safetensors
ttvidt
feature-extraction
video
video-representation-learning
self-supervised-learning
motion
temporal-modeling
dinov3
vision-transformer
custom_code
Eval Results (legacy)
Instructions to use KBlueLeaf/TTVidT with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use KBlueLeaf/TTVidT with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("video-classification", model="KBlueLeaf/TTVidT", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("KBlueLeaf/TTVidT", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Sync model code with KohakuBlueleaf/TTVidT
Browse files- modeling_ttvidt.py +16 -4
modeling_ttvidt.py
CHANGED
|
@@ -12,6 +12,8 @@ and with ``ttvidt.hub.load_model("<repo>")``: ``config.json`` holds the release
|
|
| 12 |
``model.safetensors`` uses the TTVidTrainer key names (``encoder.*``).
|
| 13 |
"""
|
| 14 |
|
|
|
|
|
|
|
| 15 |
from transformers import PretrainedConfig, PreTrainedModel
|
| 16 |
|
| 17 |
from .dinov3_vit import DINOv3VidTConfig, DINOv3VidTModel
|
|
@@ -23,8 +25,13 @@ class TTVidTConfig(PretrainedConfig):
|
|
| 23 |
|
| 24 |
model_type = "ttvidt"
|
| 25 |
|
| 26 |
-
def __init__(
|
| 27 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
self.format = format
|
| 29 |
self.model = model or {}
|
| 30 |
self.encoder_only = encoder_only
|
|
@@ -32,7 +39,9 @@ class TTVidTConfig(PretrainedConfig):
|
|
| 32 |
|
| 33 |
def encoder_config(self) -> DINOv3VidTConfig:
|
| 34 |
if self.model.get("backbone_arch", "ttvidt") != "ttvidt":
|
| 35 |
-
raise ValueError(
|
|
|
|
|
|
|
| 36 |
cfg = dict(self.model["backbone_config"])
|
| 37 |
for k, v in (self.model.get("motion_encoder_config") or {}).items():
|
| 38 |
cfg.setdefault(k, v) # same merge as TTVidTrainer
|
|
@@ -47,7 +56,10 @@ class TTVidTModel(PreTrainedModel):
|
|
| 47 |
base_model_prefix = "ttvidt"
|
| 48 |
main_input_name = "pixel_values"
|
| 49 |
# the decoder and the frame-VAE latent statistics of full releases are not used
|
| 50 |
-
_keys_to_ignore_on_load_unexpected = [
|
|
|
|
|
|
|
|
|
|
| 51 |
|
| 52 |
def __init__(self, config: TTVidTConfig):
|
| 53 |
super().__init__(config)
|
|
|
|
| 12 |
``model.safetensors`` uses the TTVidTrainer key names (``encoder.*``).
|
| 13 |
"""
|
| 14 |
|
| 15 |
+
from typing import ClassVar
|
| 16 |
+
|
| 17 |
from transformers import PretrainedConfig, PreTrainedModel
|
| 18 |
|
| 19 |
from .dinov3_vit import DINOv3VidTConfig, DINOv3VidTModel
|
|
|
|
| 25 |
|
| 26 |
model_type = "ttvidt"
|
| 27 |
|
| 28 |
+
def __init__(
|
| 29 |
+
self,
|
| 30 |
+
format: str | None = None,
|
| 31 |
+
model: dict | None = None,
|
| 32 |
+
encoder_only: bool = False,
|
| 33 |
+
**kwargs,
|
| 34 |
+
):
|
| 35 |
self.format = format
|
| 36 |
self.model = model or {}
|
| 37 |
self.encoder_only = encoder_only
|
|
|
|
| 39 |
|
| 40 |
def encoder_config(self) -> DINOv3VidTConfig:
|
| 41 |
if self.model.get("backbone_arch", "ttvidt") != "ttvidt":
|
| 42 |
+
raise ValueError(
|
| 43 |
+
f"only TT-VidT encoders are supported, got {self.model.get('backbone_arch')!r}"
|
| 44 |
+
)
|
| 45 |
cfg = dict(self.model["backbone_config"])
|
| 46 |
for k, v in (self.model.get("motion_encoder_config") or {}).items():
|
| 47 |
cfg.setdefault(k, v) # same merge as TTVidTrainer
|
|
|
|
| 56 |
base_model_prefix = "ttvidt"
|
| 57 |
main_input_name = "pixel_values"
|
| 58 |
# the decoder and the frame-VAE latent statistics of full releases are not used
|
| 59 |
+
_keys_to_ignore_on_load_unexpected: ClassVar[list[str]] = [
|
| 60 |
+
r"^decoder\.",
|
| 61 |
+
r"^latent_(mean|std)$",
|
| 62 |
+
]
|
| 63 |
|
| 64 |
def __init__(self, config: TTVidTConfig):
|
| 65 |
super().__init__(config)
|