Add OVIS 100k checkpoints (iSEE, SlotContrast) and two LA-CATER static ablations (SlotContrast on NoPE, with and without position before the MLP)

lacater_static/ablations/slotcontrast_nope/settings.yaml

@@ -0,0 +1,135 @@
+optimizer:
+  name: Adam
+  lr: 0.0004
+  lr_scheduler:
+    name: exp_decay_with_warmup
+    warmup_steps: 2500
+    decay_steps: 100000
+model:
+  initializer:
+    name: FixedLearnedInit
+    n_slots: 11
+    dim: 64
+  encoder:
+    backbone:
+      name: NoPEDinoExtractor
+      registry_name: nope_dinov2_s
+      checkpoint_path: /group/awm/pyext/dinosaw_ckpt/nope_coco_dv2_vits14_reg4.pth
+      feature: vit_output
+      frozen: true
+    output_transform:
+      name: networks.two_layer_mlp
+      inp_dim: 384
+      outp_dim: 64
+      hidden_dim: 768
+      layer_norm: true
+  grouper:
+    name: SlotAttention
+    inp_dim: 64
+    slot_dim: 64
+    n_iters: 2
+    use_mlp: true
+  decoder:
+    name: MLPDecoder
+    inp_dim: 64
+    outp_dim: 384
+    hidden_dims:
+    - 1024
+    - 1024
+    - 1024
+    n_patches: 1369
+  dynamics_predictor: null
+  predictor:
+    name: networks.TransformerEncoder
+    dim: 64
+    n_blocks: 1
+    n_heads: 4
+  target_encoder: null
+  latent_processor:
+    first_step_corrector_args:
+      n_iters: 3
+  mask_resizers: null
+  losses:
+    loss_featrec:
+      name: MSELoss
+      pred_dims:
+      - 0
+      - 384
+    loss_ss:
+      name: Slot_Slot_Contrastive_Loss
+      pred_key: processor.state
+      temperature: 0.1
+      batch_contrast: true
+      patch_inputs: false
+      keep_input_dim: true
+  loss_weights:
+    loss_featrec: 1.0
+    loss_ss: 0.5
+  input_type: video
+  target_type: features
+  target_encoder_input: null
+  encoder_input: null
+  decoder_source: state
+  aux_decoders: null
+  pointmap: null
+  cross_view: null
+  occlusion: null
+  anchor_supervision: null
+  visualize: true
+  eval_mode_config: null
+  visualize_every_n_steps: 2500
+  masks_to_visualize: null
+  load_weights: null
+  modules_to_load: null
+dataset:
+  name: LaCaterVideoDataModule
+  root: /group/awm/datasets/la_cater
+  batch_size: 32
+  chunk_size: 8
+  frame_stride: 2
+  val_chunk_size: 24
+  val_batch_size: 4
+  val_max_clips: 250
+  num_workers: 16
+  num_val_workers: 4
+  samples_per_clip: 8
+  train_transforms:
+    name: movi_train
+    type: video
+    input_size: 518
+    use_movi_normalization: false
+    h_flip_prob: 0.5
+  val_transforms:
+    name: movi_val
+    type: video
+    input_size: 518
+    use_movi_normalization: false
+    num_classes: 1
+trainer:
+  precision: bf16
+  max_steps: 100000
+  log_every_n_steps: 500
+  val_check_interval: 2500
+  gradient_clip_val: 0.05
+  accelerator: auto
+train_metrics: null
+val_metrics: null
+activation_reg_schedule: null
+depth_beta_schedule: null
+globals:
+  NUM_SLOTS: 11
+  SLOT_DIM: 64
+  DINO_MODEL: vit_small_patch14_dinov2
+  FEAT_DIM: 384
+  INPUT_SIZE: 518
+  NUM_PATCHES: 1369
+  NUM_GPUS: 1
+  BATCH_SIZE_PER_GPU: 32
+  BASE_LR: 0.0001
+experiment_name: permanence_lacater_static_sc_nope
+experiment_group: slotcontrast
+seed: 20260826
+checkpoint_every_n_steps: 1000
+checkpoint_save_top_k: 1
+checkpoint_best: null
+wandb: null

lacater_static/ablations/slotcontrast_nope/step100000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:841fcc94f267dd6a0b57ed70fc83e1fb4a925ea7ba85d8b28e0bd396012f1583
+size 123644217

lacater_static/ablations/slotcontrast_nope_pre4/settings.yaml

@@ -0,0 +1,145 @@
+optimizer:
+  name: Adam
+  lr: 0.0004
+  lr_scheduler:
+    name: exp_decay_with_warmup
+    warmup_steps: 2500
+    decay_steps: 100000
+model:
+  initializer:
+    name: FixedLearnedInit
+    n_slots: 11
+    dim: 64
+  encoder:
+    backbone:
+      name: NoPEDinoExtractor
+      registry_name: nope_dinov2_s
+      checkpoint_path: /group/awm/pyext/dinosaw_ckpt/nope_coco_dv2_vits14_reg4.pth
+      feature: vit_output
+      frozen: true
+    pre_pos_embed:
+      name: ConcatPatchPosEmbed
+      n_patches: 1369
+      mode: fourier
+      bands: 1
+      include_raw: false
+      norm_features: true
+      feature_dim: 384
+    output_transform:
+      name: networks.two_layer_mlp
+      inp_dim: 388
+      outp_dim: 64
+      hidden_dim: 768
+      layer_norm: false
+  grouper:
+    name: SlotAttention
+    inp_dim: 64
+    slot_dim: 64
+    n_iters: 2
+    use_mlp: true
+  decoder:
+    name: MLPDecoder
+    inp_dim: 64
+    outp_dim: 384
+    hidden_dims:
+    - 1024
+    - 1024
+    - 1024
+    n_patches: 1369
+  dynamics_predictor: null
+  predictor:
+    name: networks.TransformerEncoder
+    dim: 64
+    n_blocks: 1
+    n_heads: 4
+  target_encoder: null
+  latent_processor:
+    first_step_corrector_args:
+      n_iters: 3
+  mask_resizers: null
+  losses:
+    loss_featrec:
+      name: MSELoss
+      pred_dims:
+      - 0
+      - 384
+    loss_ss:
+      name: Slot_Slot_Contrastive_Loss
+      pred_key: processor.state
+      temperature: 0.1
+      batch_contrast: true
+      patch_inputs: false
+      keep_input_dim: true
+  loss_weights:
+    loss_featrec: 1.0
+    loss_ss: 0.5
+  input_type: video
+  target_type: features
+  target_encoder_input: null
+  encoder_input: null
+  decoder_source: state
+  aux_decoders: null
+  pointmap: null
+  cross_view: null
+  occlusion: null
+  anchor_supervision: null
+  visualize: true
+  eval_mode_config: null
+  visualize_every_n_steps: 2500
+  masks_to_visualize: null
+  load_weights: null
+  modules_to_load: null
+dataset:
+  name: LaCaterVideoDataModule
+  root: /group/awm/datasets/la_cater
+  batch_size: 32
+  chunk_size: 8
+  frame_stride: 2
+  val_chunk_size: 24
+  val_batch_size: 4
+  val_max_clips: 250
+  num_workers: 16
+  num_val_workers: 4
+  samples_per_clip: 8
+  train_transforms:
+    name: movi_train
+    type: video
+    input_size: 518
+    use_movi_normalization: false
+    h_flip_prob: 0.5
+  val_transforms:
+    name: movi_val
+    type: video
+    input_size: 518
+    use_movi_normalization: false
+    num_classes: 1
+trainer:
+  precision: bf16
+  max_steps: 100000
+  log_every_n_steps: 500
+  val_check_interval: 2500
+  gradient_clip_val: 0.05
+  accelerator: auto
+train_metrics: null
+val_metrics: null
+activation_reg_schedule: null
+depth_beta_schedule: null
+globals:
+  NUM_SLOTS: 11
+  SLOT_DIM: 64
+  DINO_MODEL: vit_small_patch14_dinov2
+  FEAT_DIM: 384
+  POS_DIM: 4
+  BANDS: 1
+  INPUT_SIZE: 518
+  NUM_PATCHES: 1369
+  NUM_GPUS: 1
+  BATCH_SIZE_PER_GPU: 32
+  BASE_LR: 0.0001
+experiment_name: permanence_lacater_static_sc_nope_pre4
+experiment_group: slotcontrast
+seed: 20260826
+checkpoint_every_n_steps: 1000
+checkpoint_save_top_k: 1
+checkpoint_best: null
+wandb: null

lacater_static/ablations/slotcontrast_nope_pre4/step100000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3aa5f698cc87a00ce32a3d2d60674e3ee32f3f9f21701ae7859c5abdc95b5423
+size 123681145

ovis/isee/encoder_step100000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26a309503713691c0c1eae2c03c56f556f1a571c2c79e6730cc5cb32316e706b
+size 36736481

ovis/isee/settings.yaml

@@ -0,0 +1,237 @@
+optimizer:
+  name: Adam
+  lr: 0.0004
+  lr_scheduler:
+    name: exp_decay_with_warmup
+    warmup_steps: 2500
+    decay_steps: 100000
+model:
+  initializer:
+    name: FixedLearnedInit
+    n_slots: 7
+    dim: 64
+  encoder:
+    backbone:
+      name: CachedFeatures
+      dim: 384
+      n_patches: 1369
+    output_transform:
+      name: networks.two_layer_mlp
+      inp_dim: 384
+      outp_dim: 64
+      hidden_dim: 768
+      layer_norm: true
+    post_pos_embed:
+      name: ConcatPatchPosEmbed
+      n_patches: 1369
+      mode: fourier
+      bands: 1
+      include_raw: false
+      norm_features: false
+  grouper:
+    name: SlotAttention
+    inp_dim: 68
+    slot_dim: 64
+    n_iters: 2
+    use_mlp: true
+    stream_split: 60
+    inp_split: 64
+    pos_logit_gain: 1.0
+    pos_norm: rms
+    pos_mlp: false
+    pos_init: identity
+  decoder:
+    name: CanonMLPDecoder
+    inp_dim: 64
+    outp_dim: 384
+    hidden_dims:
+    - 1024
+    - 1024
+    - 1024
+    n_patches: 1369
+    pose_dims:
+    - 60
+    - 64
+    content_dims:
+    - 0
+    - 60
+    pose_hidden: 64
+    scale_range:
+    - 0.01
+    - 4.0
+    abs_table: false
+    h_init: default
+    h_hidden_dims: []
+    fourier_bands: 8
+    delta: 5.0
+    pos_gain: 1.0
+    gate_mode: none
+  dynamics_predictor: null
+  predictor:
+    name: networks.SplitSlotPredictor
+    dim: 64
+    content_dims:
+    - 0
+    - 60
+    pose_dims:
+    - 60
+    - 64
+    appearance_blocks: 1
+    appearance_heads: 4
+    kinematics_blocks: 1
+    kinematics_heads: 4
+    content_cond_dim: 8
+    detach_content_cond: true
+  target_encoder: null
+  latent_processor:
+    first_step_corrector_args:
+      n_iters: 3
+  mask_resizers: null
+  losses:
+    loss_featrec:
+      name: MSELoss
+      pred_dims:
+      - 0
+      - 384
+    loss_ss:
+      name: Slot_Slot_Contrastive_Loss
+      pred_key: processor.state
+      temperature: 0.1
+      batch_contrast: true
+      patch_inputs: false
+      keep_input_dim: true
+      dims:
+      - 0
+      - 60
+  loss_weights:
+    loss_featrec: 1.0
+    loss_ss: 0.5
+  input_type: video
+  target_type: features
+  target_encoder_input: null
+  encoder_input: features
+  decoder_source: state
+  aux_decoders: null
+  pointmap: null
+  cross_view: null
+  occlusion: null
+  anchor_supervision: null
+  visualize: false
+  eval_mode_config: null
+  visualize_every_n_steps: 10000
+  masks_to_visualize: null
+  load_weights: null
+  modules_to_load: null
+dataset:
+  train_shards: ovis_z2c518/ovis-train-{000000..000098}.tar
+  val_shards: ovis_z2c518/ovis-dev-{000000..000015}.tar
+  batch_size: 32
+  val_batch_size: 1
+  val_size: 64
+  num_workers: 16
+  num_val_workers: 4
+  train_pipeline:
+    chunk_size: 8
+    keys:
+    - video
+    sample_one_chunk_per_video: true
+    transforms:
+      name: davis_train
+      type: video
+      crop_type: short_side_resize_random
+      input_size: 518
+      num_classes: 22
+    name: CachedViewPipeline
+    cache:
+      path: /group/awm/datasets/viewcache/ovis_z2c518/train_nope_s14_518
+      standardize: false
+      n_patches: 1369
+      dim: 384
+  val_pipeline:
+    use_chunks: false
+    keys:
+    - video
+    - segmentations
+    transforms:
+      name: davis_val
+      type: video
+      crop_type: central
+      input_size: 518
+      num_classes: 22
+    name: CachedViewPipeline
+    cache:
+      path: /group/awm/datasets/viewcache/ovis_z2c518/val_nope_s14_518
+      standardize: false
+      n_patches: 1369
+      dim: 384
+trainer:
+  max_steps: 100000
+  log_every_n_steps: 250
+  val_check_interval: 2500
+  gradient_clip_val: 0.05
+  precision: bf16
+  accelerator: auto
+train_metrics: null
+val_metrics:
+  ari:
+    name: VideoARI
+    ignore_background: false
+    pred_key: decoder_masks_hard
+    true_key: segmentations
+  image_ari:
+    name: ImageARI
+    ignore_background: false
+    video_input: true
+    pred_key: decoder_masks_hard
+    true_key: segmentations
+  mbo:
+    name: VideoIoU
+    matching: overlap
+    ignore_background: false
+    pred_key: decoder_masks_hard
+    true_key: segmentations
+  image_mbo:
+    name: ImageIoU
+    matching: overlap
+    ignore_background: false
+    video_input: true
+    pred_key: decoder_masks_hard
+    true_key: segmentations
+activation_reg_schedule: null
+depth_beta_schedule: null
+globals:
+  NUM_SLOTS: 7
+  SLOT_DIM: 64
+  DINO_MODEL: vit_small_patch14_dinov2
+  FEAT_DIM: 384
+  NUM_PATCHES: 1369
+  NUM_GPUS: 1
+  BATCH_SIZE_PER_GPU: 32
+  TOTAL_BATCH_SIZE: 32
+  BASE_LR: 0.0001
+  POS_DIM: 4
+  BANDS: 1
+  CONTENT_LO: 0
+  CONTENT_HI: 60
+  POSE_LO: 60
+  POSE_HI: 64
+experiment_name: ovis_ours_z2_k7_c8_100k_vc
+experiment_group: slotcontrast
+seed: 20260826
+checkpoint_every_n_steps: 2500
+checkpoint_save_top_k: -1
+checkpoint_best:
+  monitor: val/sel
+  mode: max
+wandb:
+  project: tsocr_table1
+  entity: pramish-paudel-insait
+  group: ovis
+  tags:
+  - ours
+  - ovis
+  - z2
+  - k7
+  - c8
+  - full
+  - vc

ovis/slotcontrast/settings.yaml

@@ -0,0 +1,188 @@
+optimizer:
+  name: Adam
+  lr: 0.0004
+  lr_scheduler:
+    name: exp_decay_with_warmup
+    warmup_steps: 2500
+    decay_steps: 100000
+model:
+  initializer:
+    name: FixedLearnedInit
+    n_slots: 7
+    dim: 64
+  encoder:
+    backbone:
+      name: CachedFeatures
+      dim: 384
+      n_patches: 1369
+    output_transform:
+      name: networks.two_layer_mlp
+      inp_dim: 384
+      outp_dim: 64
+      hidden_dim: 768
+      layer_norm: true
+  grouper:
+    name: SlotAttention
+    inp_dim: 64
+    slot_dim: 64
+    n_iters: 2
+    use_mlp: true
+  decoder:
+    name: MLPDecoder
+    inp_dim: 64
+    outp_dim: 384
+    hidden_dims:
+    - 1024
+    - 1024
+    - 1024
+    n_patches: 1369
+  dynamics_predictor: null
+  predictor:
+    name: networks.TransformerEncoder
+    dim: 64
+    n_blocks: 1
+    n_heads: 4
+  target_encoder: null
+  latent_processor:
+    first_step_corrector_args:
+      n_iters: 3
+  mask_resizers: null
+  losses:
+    loss_featrec:
+      name: MSELoss
+      pred_dims:
+      - 0
+      - 384
+    loss_ss:
+      name: Slot_Slot_Contrastive_Loss
+      pred_key: processor.state
+      temperature: 0.1
+      batch_contrast: true
+      patch_inputs: false
+      keep_input_dim: true
+  loss_weights:
+    loss_featrec: 1.0
+    loss_ss: 0.5
+  input_type: video
+  target_type: features
+  target_encoder_input: null
+  encoder_input: features
+  decoder_source: state
+  aux_decoders: null
+  pointmap: null
+  cross_view: null
+  occlusion: null
+  anchor_supervision: null
+  visualize: false
+  eval_mode_config: null
+  visualize_every_n_steps: 10000
+  masks_to_visualize: null
+  load_weights: null
+  modules_to_load: null
+dataset:
+  train_shards: ovis_z2c518/ovis-train-{000000..000098}.tar
+  val_shards: ovis_z2c518/ovis-dev-{000000..000015}.tar
+  batch_size: 32
+  val_batch_size: 1
+  val_size: 64
+  num_workers: 16
+  num_val_workers: 4
+  train_pipeline:
+    chunk_size: 8
+    keys:
+    - video
+    sample_one_chunk_per_video: true
+    transforms:
+      name: davis_train
+      type: video
+      crop_type: short_side_resize_random
+      input_size: 518
+      num_classes: 22
+    name: CachedViewPipeline
+    cache:
+      path: /group/awm/datasets/viewcache/ovis_z2c518/train_dino_s14_518
+      standardize: false
+      n_patches: 1369
+      dim: 384
+  val_pipeline:
+    use_chunks: false
+    keys:
+    - video
+    - segmentations
+    transforms:
+      name: davis_val
+      type: video
+      crop_type: central
+      input_size: 518
+      num_classes: 22
+    name: CachedViewPipeline
+    cache:
+      path: /group/awm/datasets/viewcache/ovis_z2c518/val_dino_s14_518
+      standardize: false
+      n_patches: 1369
+      dim: 384
+trainer:
+  max_steps: 100000
+  log_every_n_steps: 250
+  val_check_interval: 2500
+  gradient_clip_val: 0.05
+  precision: bf16
+  accelerator: auto
+train_metrics: null
+val_metrics:
+  ari:
+    name: VideoARI
+    ignore_background: false
+    pred_key: decoder_masks_hard
+    true_key: segmentations
+  image_ari:
+    name: ImageARI
+    ignore_background: false
+    video_input: true
+    pred_key: decoder_masks_hard
+    true_key: segmentations
+  mbo:
+    name: VideoIoU
+    matching: overlap
+    ignore_background: false
+    pred_key: decoder_masks_hard
+    true_key: segmentations
+  image_mbo:
+    name: ImageIoU
+    matching: overlap
+    ignore_background: false
+    video_input: true
+    pred_key: decoder_masks_hard
+    true_key: segmentations
+activation_reg_schedule: null
+depth_beta_schedule: null
+globals:
+  NUM_SLOTS: 7
+  SLOT_DIM: 64
+  DINO_MODEL: vit_small_patch14_dinov2
+  FEAT_DIM: 384
+  NUM_PATCHES: 1369
+  NUM_GPUS: 1
+  BATCH_SIZE_PER_GPU: 32
+  TOTAL_BATCH_SIZE: 32
+  BASE_LR: 0.0001
+experiment_name: ovis_sc_z2_k7_c8_100k_vc
+experiment_group: slotcontrast
+seed: 20260826
+checkpoint_every_n_steps: 2500
+checkpoint_save_top_k: -1
+checkpoint_best:
+  monitor: val/sel
+  mode: max
+wandb:
+  project: tsocr_table1
+  entity: pramish-paudel-insait
+  group: ovis
+  tags:
+  - sc
+  - ovis
+  - z2
+  - k7
+  - c8
+  - full
+  - vc

ovis/slotcontrast/step100000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3801c8882d97a0d269ea97ce6be7c8adfe27177872f13cb02a6be668b6965fd
+size 37438867