KBlueLeaf commited on
Commit
930b76b
·
verified ·
1 Parent(s): c9eef3f

Sync model code with KohakuBlueleaf/TTVidT

Browse files
Files changed (1) hide show
  1. modeling_ttvidt.py +16 -4
modeling_ttvidt.py CHANGED
@@ -12,6 +12,8 @@ and with ``ttvidt.hub.load_model("<repo>")``: ``config.json`` holds the release
12
  ``model.safetensors`` uses the TTVidTrainer key names (``encoder.*``).
13
  """
14
 
 
 
15
  from transformers import PretrainedConfig, PreTrainedModel
16
 
17
  from .dinov3_vit import DINOv3VidTConfig, DINOv3VidTModel
@@ -23,8 +25,13 @@ class TTVidTConfig(PretrainedConfig):
23
 
24
  model_type = "ttvidt"
25
 
26
- def __init__(self, format: str | None = None, model: dict | None = None,
27
- encoder_only: bool = False, **kwargs):
 
 
 
 
 
28
  self.format = format
29
  self.model = model or {}
30
  self.encoder_only = encoder_only
@@ -32,7 +39,9 @@ class TTVidTConfig(PretrainedConfig):
32
 
33
  def encoder_config(self) -> DINOv3VidTConfig:
34
  if self.model.get("backbone_arch", "ttvidt") != "ttvidt":
35
- raise ValueError(f"only TT-VidT encoders are supported, got {self.model.get('backbone_arch')!r}")
 
 
36
  cfg = dict(self.model["backbone_config"])
37
  for k, v in (self.model.get("motion_encoder_config") or {}).items():
38
  cfg.setdefault(k, v) # same merge as TTVidTrainer
@@ -47,7 +56,10 @@ class TTVidTModel(PreTrainedModel):
47
  base_model_prefix = "ttvidt"
48
  main_input_name = "pixel_values"
49
  # the decoder and the frame-VAE latent statistics of full releases are not used
50
- _keys_to_ignore_on_load_unexpected = [r"^decoder\.", r"^latent_(mean|std)$"]
 
 
 
51
 
52
  def __init__(self, config: TTVidTConfig):
53
  super().__init__(config)
 
12
  ``model.safetensors`` uses the TTVidTrainer key names (``encoder.*``).
13
  """
14
 
15
+ from typing import ClassVar
16
+
17
  from transformers import PretrainedConfig, PreTrainedModel
18
 
19
  from .dinov3_vit import DINOv3VidTConfig, DINOv3VidTModel
 
25
 
26
  model_type = "ttvidt"
27
 
28
+ def __init__(
29
+ self,
30
+ format: str | None = None,
31
+ model: dict | None = None,
32
+ encoder_only: bool = False,
33
+ **kwargs,
34
+ ):
35
  self.format = format
36
  self.model = model or {}
37
  self.encoder_only = encoder_only
 
39
 
40
  def encoder_config(self) -> DINOv3VidTConfig:
41
  if self.model.get("backbone_arch", "ttvidt") != "ttvidt":
42
+ raise ValueError(
43
+ f"only TT-VidT encoders are supported, got {self.model.get('backbone_arch')!r}"
44
+ )
45
  cfg = dict(self.model["backbone_config"])
46
  for k, v in (self.model.get("motion_encoder_config") or {}).items():
47
  cfg.setdefault(k, v) # same merge as TTVidTrainer
 
56
  base_model_prefix = "ttvidt"
57
  main_input_name = "pixel_values"
58
  # the decoder and the frame-VAE latent statistics of full releases are not used
59
+ _keys_to_ignore_on_load_unexpected: ClassVar[list[str]] = [
60
+ r"^decoder\.",
61
+ r"^latent_(mean|std)$",
62
+ ]
63
 
64
  def __init__(self, config: TTVidTConfig):
65
  super().__init__(config)