Archive historical metadata to free 1639 file slots

archives/filecount_cleanup_745cac8f/hydra_outputs_2026_05_to_07.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51235adbd72b93a3d6226821e07053c351faf5633a9700c201285c7a62f0713d
+size 1450750

archives/filecount_cleanup_745cac8f/wandb_segment_csv_2026_06_to_07.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c24792417027a4fc7f88ff62f3e09892dd3b8b7aa7885087634f82bb94b1fc4
+size 546687

outputs/2026-05-31/16-33-10/.hydra/config.yaml

@@ -1,235 +0,0 @@
-experiment:
-  debug: ${debug}
-  tasks:
-  - training
-  num_nodes: 1
-  training:
-    precision: 16-mixed
-    compile: false
-    lr: 2.0e-05
-    batch_size: 16
-    max_epochs: -1
-    max_steps: 350000
-    max_time: null
-    data:
-      num_workers: 4
-      shuffle: true
-    optim:
-      accumulate_grad_batches: 1
-      gradient_clip_val: 1.0
-    checkpointing:
-      every_n_train_steps: 2000
-      every_n_epochs: null
-      train_time_interval: null
-      save_last: true
-      enable_version_counter: false
-  validation:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    val_every_n_step: 2000
-    val_every_n_epoch: null
-    limit_batch: 8
-    inference_mode: true
-    data:
-      num_workers: 4
-      shuffle: false
-  test:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    limit_batch: 1
-    data:
-      num_workers: 4
-      shuffle: false
-  logging:
-    metrics: null
-dataset:
-  debug: ${debug}
-  metadata: data/${dataset.name}/metadata.json
-  data_mean: 0.5
-  data_std: 0.5
-  save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-  n_frames: 1000
-  context_length: 1
-  resolution: 128
-  observation_shape:
-  - 3
-  - ${dataset.resolution}
-  - ${dataset.resolution}
-  external_cond_dim: 0
-  validation_multiplier: 1
-  frame_skip: 1
-  action_cond_dim: 25
-  _name: video_minecraft_latent
-  precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-  n_frames_valid: 700
-  customized_validation: true
-  memory_condition_length: 0
-  wo_updown: false
-  angle_range: 180
-  pos_range: 8
-algorithm:
-  debug: ${debug}
-  lr: ${experiment.training.lr}
-  x_shape:
-  - 16
-  - 18
-  - 32
-  frame_stack: 1
-  frame_skip: ${dataset.frame_skip}
-  data_mean: ${dataset.data_mean}
-  data_std: ${dataset.data_std}
-  external_cond_dim: 0
-  context_frames: 100
-  weight_decay: 0.002
-  warmup_steps: 1000
-  optimizer_beta:
-  - 0.9
-  - 0.99
-  uncertainty_scale: 1
-  guidance_scale: 0.0
-  chunk_size: 1
-  scheduling_matrix: autoregressive
-  noise_level: random_all
-  causal: true
-  diffusion:
-    objective: pred_v
-    beta_schedule: sigmoid
-    schedule_fn_kwargs: {}
-    clip_noise: 20.0
-    use_snr: false
-    use_cum_snr: false
-    use_fused_snr: true
-    snr_clip: 5.0
-    cum_snr_decay: 0.96
-    timesteps: 1000
-    sampling_timesteps: 20
-    ddim_sampling_eta: 0.0
-    stabilization_level: 15
-    architecture:
-      network_size: 64
-      attn_heads: 4
-      attn_dim_head: 64
-      dim_mults:
-      - 1
-      - 2
-      - 4
-      - 8
-      resolution: ${dataset.resolution}
-      attn_resolutions:
-      - 16
-      - 32
-      - 64
-      - 128
-      use_init_temporal_attn: true
-      use_linear_attn: true
-      time_emb_type: rotary
-  n_frames: ${dataset.n_frames}
-  metadata: ${dataset.metadata}
-  action_cond_dim: 25
-  use_plucker: true
-  _name: dememwm_memory_dit
-  memory_token_cross_attention: true
-  memory_cross_attn_layers: null
-  memory_condition_length: 0
-  pose_cond_dim: 5
-  log_video: true
-  dememwm:
-    enabled: true
-    training_stage: stage_1
-    debug_force_all_streams: false
-    curriculum:
-      enabled: true
-      full_stage_start_step: 20000
-      freeze_vae: true
-      dit_freeze:
-        enabled: true
-      lr:
-        dememwm_modules: 4.0e-05
-        memory_adapters: 4.0e-05
-        full_dit: 1.0e-05
-    token_patch_size: 2
-    anchor:
-      enabled: true
-      anchor_indices:
-      - 0
-      - 1
-      - 2
-      - 3
-      allow_generated_as_anchor: false
-      diverse_selection: true
-      compress:
-        downsample_ratio: 3
-    dynamic:
-      enabled: true
-      exclude_latest_local_frames: 4
-      recent_frames: 4
-      conv_kernel_t: 3
-      conv_stride_t: 2
-    revisit:
-      enabled: true
-      deterministic_pose_retrieval: true
-      fov_overlap_threshold: 0.6
-      high_quality_fov_threshold: 0.7
-      plucker_weight: 0.1
-      max_frames: 2
-      fov_half_h: 52.5
-      fov_half_v: 37.5
-      fov_radius: 30.0
-      fov_yaw_samples: 25
-      fov_pitch_samples: 20
-      fov_depth_samples: 20
-      pose_preselect_topk: 64
-      plucker_grid_h: 4
-      plucker_grid_w: 4
-      plucker_focal_length: 0.35
-      compress:
-        downsample_ratio: 3
-    stage_policy:
-      noise_bucket_logging: true
-    eval_ablation:
-      enabled: false
-      branch: A_plus_D_plus_R_normal
-    generated_history_proxy:
-      enabled: true
-      start_step: 40000
-      ramp_steps: 40000
-      max_prob: 0.25
-      noise_std: 0.25
-      dropout_prob: 0.0
-    injection:
-      dit_hidden_size: 1024
-      anchor_gate: 1.0
-      dynamic_gate: 1.0
-      revisit_gate: 1.0
-    cache:
-      enabled: true
-      device: cpu
-      keep_raw_latents: all
-      keep_compressed_records: true
-      keep_prefix_anchors: true
-      eviction_policy: none
-      no_evict: true
-      clear_between_videos: true
-      max_records: null
-      on_capacity_exceeded: warn
-    checkpoint:
-      strict_dememwm_eval_load: true
-  n_tokens: 4
-debug: false
-wandb:
-  entity: turlin
-  project: DeMemWM
-  mode: offline
-resume: null
-load: null
-auto_resume: true
-resume_ckpt_path: null
-name: train_dememwm_full_berzelius_8a100_bs16_global128_350k
-output_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/
-customized_load: true
-seperate_load: true
-diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-only_tune_memory: false

outputs/2026-05-31/16-33-10/.hydra/hydra.yaml

@@ -1,232 +0,0 @@
-hydra:
-  run:
-    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
-  sweep:
-    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
-    subdir: ${hydra.job.num}
-  launcher:
-    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
-  sweeper:
-    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
-    max_batch_size: null
-    params: null
-  help:
-    app_name: ${hydra.job.name}
-    header: '${hydra.help.app_name} is powered by Hydra.
-
-      '
-    footer: 'Powered by Hydra (https://hydra.cc)
-
-      Use --hydra-help to view Hydra specific help
-
-      '
-    template: '${hydra.help.header}
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (group=option)
-
-
-      $APP_CONFIG_GROUPS
-
-
-      == Config ==
-
-      Override anything in the config (foo.bar=value)
-
-
-      $CONFIG
-
-
-      ${hydra.help.footer}
-
-      '
-  hydra_help:
-    template: 'Hydra (${hydra.runtime.version})
-
-      See https://hydra.cc for more info.
-
-
-      == Flags ==
-
-      $FLAGS_HELP
-
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
-      to command line)
-
-
-      $HYDRA_CONFIG_GROUPS
-
-
-      Use ''--cfg hydra'' to Show the Hydra config.
-
-      '
-    hydra_help: ???
-  hydra_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][HYDRA] %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-    root:
-      level: INFO
-      handlers:
-      - console
-    loggers:
-      logging_example:
-        level: DEBUG
-    disable_existing_loggers: false
-  job_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-      file:
-        class: logging.FileHandler
-        formatter: simple
-        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
-    root:
-      level: INFO
-      handlers:
-      - console
-      - file
-    disable_existing_loggers: false
-  env: {}
-  mode: RUN
-  searchpath: []
-  callbacks: {}
-  output_subdir: .hydra
-  overrides:
-    hydra:
-    - hydra.mode=RUN
-    task:
-    - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k
-    - +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/
-    - auto_resume=true
-    - experiment.tasks=[training]
-    - algorithm=dememwm_memory_dit
-    - +customized_load=true
-    - +seperate_load=true
-    - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-    - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-    - +only_tune_memory=false
-    - dataset=video_minecraft_latent
-    - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-    - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-    - dataset.n_frames=1000
-    - +dataset.n_frames_valid=700
-    - +dataset.customized_validation=true
-    - +dataset.memory_condition_length=0
-    - +dataset.wo_updown=false
-    - +dataset.angle_range=180
-    - +dataset.pos_range=8
-    - ++algorithm.n_tokens=4
-    - algorithm.x_shape=[16,18,32]
-    - ++algorithm.context_frames=100
-    - ++algorithm.log_video=true
-    - ++algorithm.diffusion.sampling_timesteps=20
-    - ++algorithm.dememwm.debug_force_all_streams=false
-    - ++algorithm.dememwm.generated_history_proxy.enabled=true
-    - ++algorithm.dememwm.generated_history_proxy.start_step=40000
-    - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-    - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-    - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-    - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-    - ++algorithm.dememwm.anchor.enabled=true
-    - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-    - ++algorithm.dememwm.anchor.diverse_selection=true
-    - ++algorithm.dememwm.anchor.compress.downsample_ratio=3
-    - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-    - ++algorithm.dememwm.dynamic.enabled=true
-    - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4
-    - ++algorithm.dememwm.dynamic.recent_frames=4
-    - ++algorithm.dememwm.revisit.enabled=true
-    - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-    - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-    - ++algorithm.dememwm.revisit.pose_preselect_topk=64
-    - ++algorithm.dememwm.revisit.fov_yaw_samples=25
-    - ++algorithm.dememwm.revisit.fov_pitch_samples=20
-    - ++algorithm.dememwm.revisit.fov_depth_samples=20
-    - ++algorithm.dememwm.revisit.plucker_weight=0.10
-    - ++algorithm.dememwm.revisit.max_frames=2
-    - ++algorithm.dememwm.revisit.compress.downsample_ratio=3
-    - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-    - ++algorithm.dememwm.cache.enabled=true
-    - ++algorithm.dememwm.cache.device=cpu
-    - ++algorithm.dememwm.cache.keep_raw_latents=all
-    - ++algorithm.dememwm.cache.keep_compressed_records=true
-    - ++algorithm.dememwm.cache.eviction_policy=none
-    - ++algorithm.dememwm.cache.no_evict=true
-    - ++algorithm.dememwm.cache.clear_between_videos=true
-    - ++algorithm.dememwm.cache.max_records=null
-    - ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-    - ++algorithm.dememwm.curriculum.enabled=true
-    - ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-    - ++algorithm.dememwm.curriculum.freeze_vae=true
-    - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-    - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-    - experiment.training.batch_size=16
-    - experiment.training.optim.accumulate_grad_batches=1
-    - experiment.validation.batch_size=1
-    - experiment.validation.limit_batch=8
-    - experiment.training.checkpointing.every_n_train_steps=2000
-    - experiment.validation.val_every_n_step=2000
-    - experiment.training.max_steps=350000
-  job:
-    name: main
-    chdir: null
-    override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=700,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=8,experiment.validation.val_every_n_step=2000
-    id: ???
-    num: ???
-    config_name: training
-    env_set: {}
-    env_copy: []
-    config:
-      override_dirname:
-        kv_sep: '='
-        item_sep: ','
-        exclude_keys: []
-  runtime:
-    version: 1.3.2
-    version_base: '1.3'
-    cwd: /proj/cvl/users/x_fahkh2/DeMemWM
-    config_sources:
-    - path: hydra.conf
-      schema: pkg
-      provider: hydra
-    - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations
-      schema: file
-      provider: main
-    - path: ''
-      schema: structured
-      provider: schema
-    output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-05-31/16-33-10
-    choices:
-      cluster: null
-      algorithm: dememwm_memory_dit
-      dataset: video_minecraft_latent
-      experiment: exp_video
-      hydra/env: default
-      hydra/callbacks: null
-      hydra/job_logging: default
-      hydra/hydra_logging: default
-      hydra/hydra_help: default
-      hydra/help: default
-      hydra/sweeper: basic
-      hydra/launcher: basic
-      hydra/output: default
-  verbose: false

outputs/2026-05-31/16-33-10/.hydra/overrides.yaml

@@ -1,74 +0,0 @@
-- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k
-- +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/
-- auto_resume=true
-- experiment.tasks=[training]
-- algorithm=dememwm_memory_dit
-- +customized_load=true
-- +seperate_load=true
-- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-- +only_tune_memory=false
-- dataset=video_minecraft_latent
-- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-- dataset.n_frames=1000
-- +dataset.n_frames_valid=700
-- +dataset.customized_validation=true
-- +dataset.memory_condition_length=0
-- +dataset.wo_updown=false
-- +dataset.angle_range=180
-- +dataset.pos_range=8
-- ++algorithm.n_tokens=4
-- algorithm.x_shape=[16,18,32]
-- ++algorithm.context_frames=100
-- ++algorithm.log_video=true
-- ++algorithm.diffusion.sampling_timesteps=20
-- ++algorithm.dememwm.debug_force_all_streams=false
-- ++algorithm.dememwm.generated_history_proxy.enabled=true
-- ++algorithm.dememwm.generated_history_proxy.start_step=40000
-- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-- ++algorithm.dememwm.anchor.enabled=true
-- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-- ++algorithm.dememwm.anchor.diverse_selection=true
-- ++algorithm.dememwm.anchor.compress.downsample_ratio=3
-- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-- ++algorithm.dememwm.dynamic.enabled=true
-- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4
-- ++algorithm.dememwm.dynamic.recent_frames=4
-- ++algorithm.dememwm.revisit.enabled=true
-- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-- ++algorithm.dememwm.revisit.pose_preselect_topk=64
-- ++algorithm.dememwm.revisit.fov_yaw_samples=25
-- ++algorithm.dememwm.revisit.fov_pitch_samples=20
-- ++algorithm.dememwm.revisit.fov_depth_samples=20
-- ++algorithm.dememwm.revisit.plucker_weight=0.10
-- ++algorithm.dememwm.revisit.max_frames=2
-- ++algorithm.dememwm.revisit.compress.downsample_ratio=3
-- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-- ++algorithm.dememwm.cache.enabled=true
-- ++algorithm.dememwm.cache.device=cpu
-- ++algorithm.dememwm.cache.keep_raw_latents=all
-- ++algorithm.dememwm.cache.keep_compressed_records=true
-- ++algorithm.dememwm.cache.eviction_policy=none
-- ++algorithm.dememwm.cache.no_evict=true
-- ++algorithm.dememwm.cache.clear_between_videos=true
-- ++algorithm.dememwm.cache.max_records=null
-- ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-- ++algorithm.dememwm.curriculum.enabled=true
-- ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-- ++algorithm.dememwm.curriculum.freeze_vae=true
-- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-- experiment.training.batch_size=16
-- experiment.training.optim.accumulate_grad_batches=1
-- experiment.validation.batch_size=1
-- experiment.validation.limit_batch=8
-- experiment.training.checkpointing.every_n_train_steps=2000
-- experiment.validation.val_every_n_step=2000
-- experiment.training.max_steps=350000

outputs/2026-05-31/16-33-10/main.log

@@ -1,38 +0,0 @@
-[2026-05-31 16:34:05,141][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8
-[2026-05-31 16:34:13,901][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8
-[2026-05-31 16:34:18,601][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8
-[2026-05-31 16:34:18,897][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8
-[2026-05-31 16:34:19,141][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8
-[2026-05-31 16:34:19,288][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8
-[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-03 16:33:00,182][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15
-[2026-06-03 16:33:00,183][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15
-[2026-06-03 16:33:00,184][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-03 16:33:00,182][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15
-[2026-06-03 16:33:00,184][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-03 16:33:00,184][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-03 16:33:00,183][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15
-[2026-06-03 16:33:00,186][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15
-[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15
-[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15
-[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15
-[2026-06-03 16:33:00,190][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-03 16:33:01,747][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15
-[2026-06-03 16:33:01,748][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-03 16:33:01,748][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15
-[2026-06-03 16:33:01,749][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15

outputs/2026-05-31/16-33-11/.hydra/config.yaml

@@ -1,235 +0,0 @@
-experiment:
-  debug: ${debug}
-  tasks:
-  - training
-  num_nodes: 1
-  training:
-    precision: 16-mixed
-    compile: false
-    lr: 2.0e-05
-    batch_size: 16
-    max_epochs: -1
-    max_steps: 350000
-    max_time: null
-    data:
-      num_workers: 4
-      shuffle: true
-    optim:
-      accumulate_grad_batches: 1
-      gradient_clip_val: 1.0
-    checkpointing:
-      every_n_train_steps: 2000
-      every_n_epochs: null
-      train_time_interval: null
-      save_last: true
-      enable_version_counter: false
-  validation:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    val_every_n_step: 2000
-    val_every_n_epoch: null
-    limit_batch: 8
-    inference_mode: true
-    data:
-      num_workers: 4
-      shuffle: false
-  test:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    limit_batch: 1
-    data:
-      num_workers: 4
-      shuffle: false
-  logging:
-    metrics: null
-dataset:
-  debug: ${debug}
-  metadata: data/${dataset.name}/metadata.json
-  data_mean: 0.5
-  data_std: 0.5
-  save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-  n_frames: 1000
-  context_length: 1
-  resolution: 128
-  observation_shape:
-  - 3
-  - ${dataset.resolution}
-  - ${dataset.resolution}
-  external_cond_dim: 0
-  validation_multiplier: 1
-  frame_skip: 1
-  action_cond_dim: 25
-  _name: video_minecraft_latent
-  precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-  n_frames_valid: 700
-  customized_validation: true
-  memory_condition_length: 0
-  wo_updown: false
-  angle_range: 180
-  pos_range: 8
-algorithm:
-  debug: ${debug}
-  lr: ${experiment.training.lr}
-  x_shape:
-  - 16
-  - 18
-  - 32
-  frame_stack: 1
-  frame_skip: ${dataset.frame_skip}
-  data_mean: ${dataset.data_mean}
-  data_std: ${dataset.data_std}
-  external_cond_dim: 0
-  context_frames: 100
-  weight_decay: 0.002
-  warmup_steps: 1000
-  optimizer_beta:
-  - 0.9
-  - 0.99
-  uncertainty_scale: 1
-  guidance_scale: 0.0
-  chunk_size: 1
-  scheduling_matrix: autoregressive
-  noise_level: random_all
-  causal: true
-  diffusion:
-    objective: pred_v
-    beta_schedule: sigmoid
-    schedule_fn_kwargs: {}
-    clip_noise: 20.0
-    use_snr: false
-    use_cum_snr: false
-    use_fused_snr: true
-    snr_clip: 5.0
-    cum_snr_decay: 0.96
-    timesteps: 1000
-    sampling_timesteps: 20
-    ddim_sampling_eta: 0.0
-    stabilization_level: 15
-    architecture:
-      network_size: 64
-      attn_heads: 4
-      attn_dim_head: 64
-      dim_mults:
-      - 1
-      - 2
-      - 4
-      - 8
-      resolution: ${dataset.resolution}
-      attn_resolutions:
-      - 16
-      - 32
-      - 64
-      - 128
-      use_init_temporal_attn: true
-      use_linear_attn: true
-      time_emb_type: rotary
-  n_frames: ${dataset.n_frames}
-  metadata: ${dataset.metadata}
-  action_cond_dim: 25
-  use_plucker: true
-  _name: dememwm_memory_dit
-  memory_token_cross_attention: true
-  memory_cross_attn_layers: null
-  memory_condition_length: 0
-  pose_cond_dim: 5
-  log_video: true
-  dememwm:
-    enabled: true
-    training_stage: stage_1
-    debug_force_all_streams: false
-    curriculum:
-      enabled: true
-      full_stage_start_step: 20000
-      freeze_vae: true
-      dit_freeze:
-        enabled: true
-      lr:
-        dememwm_modules: 4.0e-05
-        memory_adapters: 4.0e-05
-        full_dit: 1.0e-05
-    token_patch_size: 2
-    anchor:
-      enabled: true
-      anchor_indices:
-      - 0
-      - 1
-      - 2
-      - 3
-      allow_generated_as_anchor: false
-      diverse_selection: true
-      compress:
-        downsample_ratio: 3
-    dynamic:
-      enabled: true
-      exclude_latest_local_frames: 4
-      recent_frames: 4
-      conv_kernel_t: 3
-      conv_stride_t: 2
-    revisit:
-      enabled: true
-      deterministic_pose_retrieval: true
-      fov_overlap_threshold: 0.6
-      high_quality_fov_threshold: 0.7
-      plucker_weight: 0.1
-      max_frames: 2
-      fov_half_h: 52.5
-      fov_half_v: 37.5
-      fov_radius: 30.0
-      fov_yaw_samples: 25
-      fov_pitch_samples: 20
-      fov_depth_samples: 20
-      pose_preselect_topk: 64
-      plucker_grid_h: 4
-      plucker_grid_w: 4
-      plucker_focal_length: 0.35
-      compress:
-        downsample_ratio: 3
-    stage_policy:
-      noise_bucket_logging: true
-    eval_ablation:
-      enabled: false
-      branch: A_plus_D_plus_R_normal
-    generated_history_proxy:
-      enabled: true
-      start_step: 40000
-      ramp_steps: 40000
-      max_prob: 0.25
-      noise_std: 0.25
-      dropout_prob: 0.0
-    injection:
-      dit_hidden_size: 1024
-      anchor_gate: 1.0
-      dynamic_gate: 1.0
-      revisit_gate: 1.0
-    cache:
-      enabled: true
-      device: cpu
-      keep_raw_latents: all
-      keep_compressed_records: true
-      keep_prefix_anchors: true
-      eviction_policy: none
-      no_evict: true
-      clear_between_videos: true
-      max_records: null
-      on_capacity_exceeded: warn
-    checkpoint:
-      strict_dememwm_eval_load: true
-  n_tokens: 4
-debug: false
-wandb:
-  entity: turlin
-  project: DeMemWM
-  mode: offline
-resume: null
-load: null
-auto_resume: true
-resume_ckpt_path: null
-name: train_dememwm_full_berzelius_8a100_bs16_global128_350k
-output_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/
-customized_load: true
-seperate_load: true
-diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-only_tune_memory: false

outputs/2026-05-31/16-33-11/.hydra/hydra.yaml

@@ -1,232 +0,0 @@
-hydra:
-  run:
-    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
-  sweep:
-    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
-    subdir: ${hydra.job.num}
-  launcher:
-    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
-  sweeper:
-    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
-    max_batch_size: null
-    params: null
-  help:
-    app_name: ${hydra.job.name}
-    header: '${hydra.help.app_name} is powered by Hydra.
-
-      '
-    footer: 'Powered by Hydra (https://hydra.cc)
-
-      Use --hydra-help to view Hydra specific help
-
-      '
-    template: '${hydra.help.header}
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (group=option)
-
-
-      $APP_CONFIG_GROUPS
-
-
-      == Config ==
-
-      Override anything in the config (foo.bar=value)
-
-
-      $CONFIG
-
-
-      ${hydra.help.footer}
-
-      '
-  hydra_help:
-    template: 'Hydra (${hydra.runtime.version})
-
-      See https://hydra.cc for more info.
-
-
-      == Flags ==
-
-      $FLAGS_HELP
-
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
-      to command line)
-
-
-      $HYDRA_CONFIG_GROUPS
-
-
-      Use ''--cfg hydra'' to Show the Hydra config.
-
-      '
-    hydra_help: ???
-  hydra_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][HYDRA] %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-    root:
-      level: INFO
-      handlers:
-      - console
-    loggers:
-      logging_example:
-        level: DEBUG
-    disable_existing_loggers: false
-  job_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-      file:
-        class: logging.FileHandler
-        formatter: simple
-        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
-    root:
-      level: INFO
-      handlers:
-      - console
-      - file
-    disable_existing_loggers: false
-  env: {}
-  mode: RUN
-  searchpath: []
-  callbacks: {}
-  output_subdir: .hydra
-  overrides:
-    hydra:
-    - hydra.mode=RUN
-    task:
-    - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k
-    - +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/
-    - auto_resume=true
-    - experiment.tasks=[training]
-    - algorithm=dememwm_memory_dit
-    - +customized_load=true
-    - +seperate_load=true
-    - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-    - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-    - +only_tune_memory=false
-    - dataset=video_minecraft_latent
-    - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-    - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-    - dataset.n_frames=1000
-    - +dataset.n_frames_valid=700
-    - +dataset.customized_validation=true
-    - +dataset.memory_condition_length=0
-    - +dataset.wo_updown=false
-    - +dataset.angle_range=180
-    - +dataset.pos_range=8
-    - ++algorithm.n_tokens=4
-    - algorithm.x_shape=[16,18,32]
-    - ++algorithm.context_frames=100
-    - ++algorithm.log_video=true
-    - ++algorithm.diffusion.sampling_timesteps=20
-    - ++algorithm.dememwm.debug_force_all_streams=false
-    - ++algorithm.dememwm.generated_history_proxy.enabled=true
-    - ++algorithm.dememwm.generated_history_proxy.start_step=40000
-    - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-    - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-    - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-    - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-    - ++algorithm.dememwm.anchor.enabled=true
-    - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-    - ++algorithm.dememwm.anchor.diverse_selection=true
-    - ++algorithm.dememwm.anchor.compress.downsample_ratio=3
-    - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-    - ++algorithm.dememwm.dynamic.enabled=true
-    - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4
-    - ++algorithm.dememwm.dynamic.recent_frames=4
-    - ++algorithm.dememwm.revisit.enabled=true
-    - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-    - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-    - ++algorithm.dememwm.revisit.pose_preselect_topk=64
-    - ++algorithm.dememwm.revisit.fov_yaw_samples=25
-    - ++algorithm.dememwm.revisit.fov_pitch_samples=20
-    - ++algorithm.dememwm.revisit.fov_depth_samples=20
-    - ++algorithm.dememwm.revisit.plucker_weight=0.10
-    - ++algorithm.dememwm.revisit.max_frames=2
-    - ++algorithm.dememwm.revisit.compress.downsample_ratio=3
-    - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-    - ++algorithm.dememwm.cache.enabled=true
-    - ++algorithm.dememwm.cache.device=cpu
-    - ++algorithm.dememwm.cache.keep_raw_latents=all
-    - ++algorithm.dememwm.cache.keep_compressed_records=true
-    - ++algorithm.dememwm.cache.eviction_policy=none
-    - ++algorithm.dememwm.cache.no_evict=true
-    - ++algorithm.dememwm.cache.clear_between_videos=true
-    - ++algorithm.dememwm.cache.max_records=null
-    - ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-    - ++algorithm.dememwm.curriculum.enabled=true
-    - ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-    - ++algorithm.dememwm.curriculum.freeze_vae=true
-    - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-    - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-    - experiment.training.batch_size=16
-    - experiment.training.optim.accumulate_grad_batches=1
-    - experiment.validation.batch_size=1
-    - experiment.validation.limit_batch=8
-    - experiment.training.checkpointing.every_n_train_steps=2000
-    - experiment.validation.val_every_n_step=2000
-    - experiment.training.max_steps=350000
-  job:
-    name: main
-    chdir: null
-    override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=700,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=8,experiment.validation.val_every_n_step=2000
-    id: ???
-    num: ???
-    config_name: training
-    env_set: {}
-    env_copy: []
-    config:
-      override_dirname:
-        kv_sep: '='
-        item_sep: ','
-        exclude_keys: []
-  runtime:
-    version: 1.3.2
-    version_base: '1.3'
-    cwd: /proj/cvl/users/x_fahkh2/DeMemWM
-    config_sources:
-    - path: hydra.conf
-      schema: pkg
-      provider: hydra
-    - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations
-      schema: file
-      provider: main
-    - path: ''
-      schema: structured
-      provider: schema
-    output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-05-31/16-33-11
-    choices:
-      cluster: null
-      algorithm: dememwm_memory_dit
-      dataset: video_minecraft_latent
-      experiment: exp_video
-      hydra/env: default
-      hydra/callbacks: null
-      hydra/job_logging: default
-      hydra/hydra_logging: default
-      hydra/hydra_help: default
-      hydra/help: default
-      hydra/sweeper: basic
-      hydra/launcher: basic
-      hydra/output: default
-  verbose: false

outputs/2026-05-31/16-33-11/.hydra/overrides.yaml

@@ -1,74 +0,0 @@
-- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k
-- +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/
-- auto_resume=true
-- experiment.tasks=[training]
-- algorithm=dememwm_memory_dit
-- +customized_load=true
-- +seperate_load=true
-- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-- +only_tune_memory=false
-- dataset=video_minecraft_latent
-- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-- dataset.n_frames=1000
-- +dataset.n_frames_valid=700
-- +dataset.customized_validation=true
-- +dataset.memory_condition_length=0
-- +dataset.wo_updown=false
-- +dataset.angle_range=180
-- +dataset.pos_range=8
-- ++algorithm.n_tokens=4
-- algorithm.x_shape=[16,18,32]
-- ++algorithm.context_frames=100
-- ++algorithm.log_video=true
-- ++algorithm.diffusion.sampling_timesteps=20
-- ++algorithm.dememwm.debug_force_all_streams=false
-- ++algorithm.dememwm.generated_history_proxy.enabled=true
-- ++algorithm.dememwm.generated_history_proxy.start_step=40000
-- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-- ++algorithm.dememwm.anchor.enabled=true
-- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-- ++algorithm.dememwm.anchor.diverse_selection=true
-- ++algorithm.dememwm.anchor.compress.downsample_ratio=3
-- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-- ++algorithm.dememwm.dynamic.enabled=true
-- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4
-- ++algorithm.dememwm.dynamic.recent_frames=4
-- ++algorithm.dememwm.revisit.enabled=true
-- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-- ++algorithm.dememwm.revisit.pose_preselect_topk=64
-- ++algorithm.dememwm.revisit.fov_yaw_samples=25
-- ++algorithm.dememwm.revisit.fov_pitch_samples=20
-- ++algorithm.dememwm.revisit.fov_depth_samples=20
-- ++algorithm.dememwm.revisit.plucker_weight=0.10
-- ++algorithm.dememwm.revisit.max_frames=2
-- ++algorithm.dememwm.revisit.compress.downsample_ratio=3
-- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-- ++algorithm.dememwm.cache.enabled=true
-- ++algorithm.dememwm.cache.device=cpu
-- ++algorithm.dememwm.cache.keep_raw_latents=all
-- ++algorithm.dememwm.cache.keep_compressed_records=true
-- ++algorithm.dememwm.cache.eviction_policy=none
-- ++algorithm.dememwm.cache.no_evict=true
-- ++algorithm.dememwm.cache.clear_between_videos=true
-- ++algorithm.dememwm.cache.max_records=null
-- ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-- ++algorithm.dememwm.curriculum.enabled=true
-- ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-- ++algorithm.dememwm.curriculum.freeze_vae=true
-- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-- experiment.training.batch_size=16
-- experiment.training.optim.accumulate_grad_batches=1
-- experiment.validation.batch_size=1
-- experiment.validation.limit_batch=8
-- experiment.training.checkpointing.every_n_train_steps=2000
-- experiment.validation.val_every_n_step=2000
-- experiment.training.max_steps=350000

outputs/2026-05-31/16-33-11/main.log

@@ -1,39 +0,0 @@
-[2026-05-31 16:33:58,507][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP)
-[2026-05-31 16:33:58,787][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True
-[2026-05-31 16:33:58,787][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores
-[2026-05-31 16:33:58,787][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs
-[2026-05-31 16:33:58,787][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs
-[2026-05-31 16:34:01,148][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix=''
-[2026-05-31 16:34:02,180][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix=''
-[2026-05-31 16:34:05,142][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8
-[2026-05-31 16:34:14,613][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8
-[2026-05-31 16:34:20,380][pytorch_lightning.utilities.rank_zero][INFO] - ----------------------------------------------------------------------------------------------------
-distributed_backend=nccl
-All distributed processes registered. Starting with 8 processes
-----------------------------------------------------------------------------------------------------
-
-[2026-05-31 16:34:34,882][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-05-31 16:34:36,691][lightning.pytorch.callbacks.model_summary][INFO] - 
-  | Name                       | Type                                  | Params
--------------------------------------------------------------------------------------
-0 | diffusion_model            | Diffusion                             | 910 M 
-1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor         | 199 K 
-2 | dememwm_anchor_proj        | SpatialConv2DMemoryProjector          | 591 K 
-3 | dememwm_revisit_proj       | SpatialConv2DMemoryProjector          | 591 K 
-4 | dememwm_revisit_gate       | RevisitRawGate                        | 4     
-5 | validation_lpips_model     | LearnedPerceptualImagePatchSimilarity | 2.5 M 
-6 | vae                        | AutoencoderKL                         | 229 M 
--------------------------------------------------------------------------------------
-911 M     Trainable params
-231 M     Non-trainable params
-1.1 B     Total params
-4,573.841 Total estimated model params size (MB)
-[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-03 16:33:00,183][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15
-[2026-06-03 16:33:00,184][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15
-[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-03 16:33:01,749][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15
-[2026-06-03 16:33:01,750][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15

outputs/2026-06-01/15-41-01/.hydra/config.yaml

@@ -1,234 +0,0 @@
-experiment:
-  debug: ${debug}
-  tasks:
-  - training
-  num_nodes: 1
-  training:
-    precision: 16-mixed
-    compile: false
-    lr: 2.0e-05
-    batch_size: 16
-    max_epochs: -1
-    max_steps: 350000
-    max_time: null
-    data:
-      num_workers: 4
-      shuffle: true
-    optim:
-      accumulate_grad_batches: 1
-      gradient_clip_val: 1.0
-    checkpointing:
-      every_n_train_steps: 2000
-      every_n_epochs: null
-      train_time_interval: null
-      save_last: true
-      enable_version_counter: false
-  validation:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    val_every_n_step: 2000
-    val_every_n_epoch: null
-    limit_batch: 2
-    inference_mode: true
-    data:
-      num_workers: 4
-      shuffle: false
-  test:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    limit_batch: 1
-    data:
-      num_workers: 4
-      shuffle: false
-  logging:
-    metrics: null
-dataset:
-  debug: ${debug}
-  metadata: data/${dataset.name}/metadata.json
-  data_mean: 0.5
-  data_std: 0.5
-  save_dir: /share_1/users/bonan_ding/worldmem_data/minecraft
-  n_frames: 1000
-  context_length: 1
-  resolution: 128
-  observation_shape:
-  - 3
-  - ${dataset.resolution}
-  - ${dataset.resolution}
-  external_cond_dim: 0
-  validation_multiplier: 1
-  frame_skip: 1
-  action_cond_dim: 25
-  _name: video_minecraft_latent
-  precomputed_feature_dir: /share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
-  n_frames_valid: 1100
-  customized_validation: true
-  wo_updown: false
-  angle_range: 180
-  pos_range: 8
-algorithm:
-  debug: ${debug}
-  lr: ${experiment.training.lr}
-  x_shape:
-  - 16
-  - 18
-  - 32
-  frame_stack: 1
-  frame_skip: ${dataset.frame_skip}
-  data_mean: ${dataset.data_mean}
-  data_std: ${dataset.data_std}
-  external_cond_dim: 0
-  context_frames: 100
-  weight_decay: 0.002
-  warmup_steps: 1000
-  optimizer_beta:
-  - 0.9
-  - 0.99
-  uncertainty_scale: 1
-  guidance_scale: 0.0
-  chunk_size: 1
-  scheduling_matrix: autoregressive
-  noise_level: random_all
-  causal: true
-  diffusion:
-    objective: pred_v
-    beta_schedule: sigmoid
-    schedule_fn_kwargs: {}
-    clip_noise: 20.0
-    use_snr: false
-    use_cum_snr: false
-    use_fused_snr: true
-    snr_clip: 5.0
-    cum_snr_decay: 0.96
-    timesteps: 1000
-    sampling_timesteps: 20
-    ddim_sampling_eta: 0.0
-    stabilization_level: 15
-    architecture:
-      network_size: 64
-      attn_heads: 4
-      attn_dim_head: 64
-      dim_mults:
-      - 1
-      - 2
-      - 4
-      - 8
-      resolution: ${dataset.resolution}
-      attn_resolutions:
-      - 16
-      - 32
-      - 64
-      - 128
-      use_init_temporal_attn: true
-      use_linear_attn: true
-      time_emb_type: rotary
-  n_frames: ${dataset.n_frames}
-  metadata: ${dataset.metadata}
-  action_cond_dim: 25
-  use_plucker: true
-  _name: dememwm_memory_dit
-  memory_token_cross_attention: true
-  memory_cross_attn_layers:
-  - 3
-  - 7
-  - 11
-  - 15
-  pose_cond_dim: 5
-  log_video: true
-  dememwm:
-    enabled: true
-    training_stage: stage_1
-    debug_force_all_streams: false
-    curriculum:
-      enabled: true
-      full_stage_start_step: 20000
-      freeze_vae: true
-      dit_freeze:
-        enabled: true
-      lr:
-        dememwm_modules: 4.0e-05
-        memory_adapters: 4.0e-05
-        full_dit: 1.0e-05
-    spatial_kernel_size: 3
-    anchor:
-      enabled: true
-      anchor_indices:
-      - 0
-      - 1
-      - 2
-      - 3
-      allow_generated_as_anchor: false
-      diverse_selection: true
-      compress:
-        downsample_ratio: 2
-    dynamic:
-      enabled: true
-      recent_frames: 4
-      conv_kernel_t: 3
-      conv_stride_t: 2
-      compress:
-        downsample_ratio: 2
-    revisit:
-      enabled: true
-      deterministic_pose_retrieval: true
-      fov_overlap_threshold: 0.6
-      plucker_weight: 0.1
-      max_frames: 2
-      fov_half_h: 52.5
-      fov_half_v: 37.5
-      fov_radius: 30.0
-      fov_yaw_samples: 25
-      fov_pitch_samples: 20
-      fov_depth_samples: 20
-      pose_preselect_topk: 64
-      plucker_grid_h: 4
-      plucker_grid_w: 4
-      plucker_focal_length: 0.35
-      compress:
-        downsample_ratio: 2
-    stage_policy:
-      noise_bucket_logging: true
-    eval_ablation:
-      enabled: false
-      branch: A_plus_D_plus_R_normal
-    generated_history_proxy:
-      enabled: true
-      start_step: 40000
-      ramp_steps: 40000
-      max_prob: 0.25
-      noise_std: 0.25
-      dropout_prob: 0.0
-    injection:
-      memory_hidden_size: 512
-    cache:
-      enabled: true
-      device: cpu
-      keep_raw_latents: all
-      keep_compressed_records: true
-      keep_prefix_anchors: true
-      eviction_policy: none
-      no_evict: true
-      clear_between_videos: true
-      max_records: null
-      on_capacity_exceeded: warn
-    checkpoint:
-      strict_dememwm_eval_load: true
-  n_tokens: 4
-debug: false
-wandb:
-  entity: turlin
-  project: worldmem
-  mode: offline
-resume: null
-load: null
-auto_resume: true
-resume_ckpt_path: null
-name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-output_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-customized_load: true
-seperate_load: true
-diffusion_model_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors
-vae_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors
-only_tune_memory: false

outputs/2026-06-01/15-41-01/.hydra/hydra.yaml

@@ -1,232 +0,0 @@
-hydra:
-  run:
-    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
-  sweep:
-    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
-    subdir: ${hydra.job.num}
-  launcher:
-    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
-  sweeper:
-    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
-    max_batch_size: null
-    params: null
-  help:
-    app_name: ${hydra.job.name}
-    header: '${hydra.help.app_name} is powered by Hydra.
-
-      '
-    footer: 'Powered by Hydra (https://hydra.cc)
-
-      Use --hydra-help to view Hydra specific help
-
-      '
-    template: '${hydra.help.header}
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (group=option)
-
-
-      $APP_CONFIG_GROUPS
-
-
-      == Config ==
-
-      Override anything in the config (foo.bar=value)
-
-
-      $CONFIG
-
-
-      ${hydra.help.footer}
-
-      '
-  hydra_help:
-    template: 'Hydra (${hydra.runtime.version})
-
-      See https://hydra.cc for more info.
-
-
-      == Flags ==
-
-      $FLAGS_HELP
-
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
-      to command line)
-
-
-      $HYDRA_CONFIG_GROUPS
-
-
-      Use ''--cfg hydra'' to Show the Hydra config.
-
-      '
-    hydra_help: ???
-  hydra_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][HYDRA] %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-    root:
-      level: INFO
-      handlers:
-      - console
-    loggers:
-      logging_example:
-        level: DEBUG
-    disable_existing_loggers: false
-  job_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-      file:
-        class: logging.FileHandler
-        formatter: simple
-        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
-    root:
-      level: INFO
-      handlers:
-      - console
-      - file
-    disable_existing_loggers: false
-  env: {}
-  mode: RUN
-  searchpath: []
-  callbacks: {}
-  output_subdir: .hydra
-  overrides:
-    hydra:
-    - hydra.mode=RUN
-    task:
-    - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-    - +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-    - auto_resume=true
-    - experiment.tasks=[training]
-    - algorithm=dememwm_memory_dit
-    - +customized_load=true
-    - +seperate_load=true
-    - +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors
-    - +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors
-    - +only_tune_memory=false
-    - dataset=video_minecraft_latent
-    - dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft
-    - dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
-    - dataset.n_frames=1000
-    - +dataset.n_frames_valid=1100
-    - +dataset.customized_validation=true
-    - +dataset.wo_updown=false
-    - +dataset.angle_range=180
-    - +dataset.pos_range=8
-    - ++algorithm.n_tokens=4
-    - algorithm.x_shape=[16,18,32]
-    - ++algorithm.context_frames=100
-    - ++algorithm.log_video=true
-    - ++algorithm.diffusion.sampling_timesteps=20
-    - ++algorithm.dememwm.debug_force_all_streams=false
-    - ++algorithm.dememwm.generated_history_proxy.enabled=true
-    - ++algorithm.dememwm.generated_history_proxy.start_step=40000
-    - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-    - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-    - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-    - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-    - ++algorithm.dememwm.anchor.enabled=true
-    - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-    - ++algorithm.dememwm.anchor.diverse_selection=true
-    - ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-    - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-    - ++algorithm.dememwm.dynamic.enabled=true
-    - ++algorithm.dememwm.dynamic.recent_frames=4
-    - ++algorithm.dememwm.dynamic.conv_stride_t=2
-    - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-    - ++algorithm.dememwm.revisit.enabled=true
-    - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-    - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-    - ++algorithm.dememwm.revisit.pose_preselect_topk=64
-    - ++algorithm.dememwm.revisit.fov_yaw_samples=25
-    - ++algorithm.dememwm.revisit.fov_pitch_samples=20
-    - ++algorithm.dememwm.revisit.fov_depth_samples=20
-    - ++algorithm.dememwm.revisit.plucker_weight=0.10
-    - ++algorithm.dememwm.revisit.max_frames=2
-    - ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-    - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-    - ++algorithm.dememwm.cache.enabled=true
-    - ++algorithm.dememwm.cache.device=cpu
-    - ++algorithm.dememwm.cache.keep_raw_latents=all
-    - ++algorithm.dememwm.cache.keep_compressed_records=true
-    - ++algorithm.dememwm.cache.eviction_policy=none
-    - ++algorithm.dememwm.cache.no_evict=true
-    - ++algorithm.dememwm.cache.clear_between_videos=true
-    - ++algorithm.dememwm.cache.max_records=null
-    - ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-    - ++algorithm.dememwm.curriculum.enabled=true
-    - ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-    - ++algorithm.dememwm.curriculum.freeze_vae=true
-    - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-    - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-    - experiment.training.batch_size=16
-    - experiment.training.optim.accumulate_grad_batches=1
-    - experiment.validation.batch_size=1
-    - experiment.validation.limit_batch=2
-    - experiment.training.checkpointing.every_n_train_steps=2000
-    - experiment.validation.val_every_n_step=2000
-    - experiment.training.max_steps=350000
-  job:
-    name: main
-    chdir: null
-    override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features,dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000
-    id: ???
-    num: ???
-    config_name: training
-    env_set: {}
-    env_copy: []
-    config:
-      override_dirname:
-        kv_sep: '='
-        item_sep: ','
-        exclude_keys: []
-  runtime:
-    version: 1.3.2
-    version_base: '1.3'
-    cwd: /proj/cvl/users/x_fahkh2/DeMemWM
-    config_sources:
-    - path: hydra.conf
-      schema: pkg
-      provider: hydra
-    - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations
-      schema: file
-      provider: main
-    - path: ''
-      schema: structured
-      provider: schema
-    output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-01/15-41-01
-    choices:
-      cluster: null
-      algorithm: dememwm_memory_dit
-      dataset: video_minecraft_latent
-      experiment: exp_video
-      hydra/env: default
-      hydra/callbacks: null
-      hydra/job_logging: default
-      hydra/hydra_logging: default
-      hydra/hydra_help: default
-      hydra/help: default
-      hydra/sweeper: basic
-      hydra/launcher: basic
-      hydra/output: default
-  verbose: false

outputs/2026-06-01/15-41-01/.hydra/overrides.yaml

@@ -1,74 +0,0 @@
-- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-- +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-- auto_resume=true
-- experiment.tasks=[training]
-- algorithm=dememwm_memory_dit
-- +customized_load=true
-- +seperate_load=true
-- +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors
-- +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors
-- +only_tune_memory=false
-- dataset=video_minecraft_latent
-- dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft
-- dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
-- dataset.n_frames=1000
-- +dataset.n_frames_valid=1100
-- +dataset.customized_validation=true
-- +dataset.wo_updown=false
-- +dataset.angle_range=180
-- +dataset.pos_range=8
-- ++algorithm.n_tokens=4
-- algorithm.x_shape=[16,18,32]
-- ++algorithm.context_frames=100
-- ++algorithm.log_video=true
-- ++algorithm.diffusion.sampling_timesteps=20
-- ++algorithm.dememwm.debug_force_all_streams=false
-- ++algorithm.dememwm.generated_history_proxy.enabled=true
-- ++algorithm.dememwm.generated_history_proxy.start_step=40000
-- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-- ++algorithm.dememwm.anchor.enabled=true
-- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-- ++algorithm.dememwm.anchor.diverse_selection=true
-- ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-- ++algorithm.dememwm.dynamic.enabled=true
-- ++algorithm.dememwm.dynamic.recent_frames=4
-- ++algorithm.dememwm.dynamic.conv_stride_t=2
-- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-- ++algorithm.dememwm.revisit.enabled=true
-- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-- ++algorithm.dememwm.revisit.pose_preselect_topk=64
-- ++algorithm.dememwm.revisit.fov_yaw_samples=25
-- ++algorithm.dememwm.revisit.fov_pitch_samples=20
-- ++algorithm.dememwm.revisit.fov_depth_samples=20
-- ++algorithm.dememwm.revisit.plucker_weight=0.10
-- ++algorithm.dememwm.revisit.max_frames=2
-- ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-- ++algorithm.dememwm.cache.enabled=true
-- ++algorithm.dememwm.cache.device=cpu
-- ++algorithm.dememwm.cache.keep_raw_latents=all
-- ++algorithm.dememwm.cache.keep_compressed_records=true
-- ++algorithm.dememwm.cache.eviction_policy=none
-- ++algorithm.dememwm.cache.no_evict=true
-- ++algorithm.dememwm.cache.clear_between_videos=true
-- ++algorithm.dememwm.cache.max_records=null
-- ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-- ++algorithm.dememwm.curriculum.enabled=true
-- ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-- ++algorithm.dememwm.curriculum.freeze_vae=true
-- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-- experiment.training.batch_size=16
-- experiment.training.optim.accumulate_grad_batches=1
-- experiment.validation.batch_size=1
-- experiment.validation.limit_batch=2
-- experiment.training.checkpointing.every_n_train_steps=2000
-- experiment.validation.val_every_n_step=2000
-- experiment.training.max_steps=350000

outputs/2026-06-01/15-41-06/.hydra/config.yaml

@@ -1,234 +0,0 @@
-experiment:
-  debug: ${debug}
-  tasks:
-  - training
-  num_nodes: 1
-  training:
-    precision: 16-mixed
-    compile: false
-    lr: 2.0e-05
-    batch_size: 16
-    max_epochs: -1
-    max_steps: 350000
-    max_time: null
-    data:
-      num_workers: 4
-      shuffle: true
-    optim:
-      accumulate_grad_batches: 1
-      gradient_clip_val: 1.0
-    checkpointing:
-      every_n_train_steps: 2000
-      every_n_epochs: null
-      train_time_interval: null
-      save_last: true
-      enable_version_counter: false
-  validation:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    val_every_n_step: 2000
-    val_every_n_epoch: null
-    limit_batch: 2
-    inference_mode: true
-    data:
-      num_workers: 4
-      shuffle: false
-  test:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    limit_batch: 1
-    data:
-      num_workers: 4
-      shuffle: false
-  logging:
-    metrics: null
-dataset:
-  debug: ${debug}
-  metadata: data/${dataset.name}/metadata.json
-  data_mean: 0.5
-  data_std: 0.5
-  save_dir: /share_1/users/bonan_ding/worldmem_data/minecraft
-  n_frames: 1000
-  context_length: 1
-  resolution: 128
-  observation_shape:
-  - 3
-  - ${dataset.resolution}
-  - ${dataset.resolution}
-  external_cond_dim: 0
-  validation_multiplier: 1
-  frame_skip: 1
-  action_cond_dim: 25
-  _name: video_minecraft_latent
-  precomputed_feature_dir: /share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
-  n_frames_valid: 1100
-  customized_validation: true
-  wo_updown: false
-  angle_range: 180
-  pos_range: 8
-algorithm:
-  debug: ${debug}
-  lr: ${experiment.training.lr}
-  x_shape:
-  - 16
-  - 18
-  - 32
-  frame_stack: 1
-  frame_skip: ${dataset.frame_skip}
-  data_mean: ${dataset.data_mean}
-  data_std: ${dataset.data_std}
-  external_cond_dim: 0
-  context_frames: 100
-  weight_decay: 0.002
-  warmup_steps: 1000
-  optimizer_beta:
-  - 0.9
-  - 0.99
-  uncertainty_scale: 1
-  guidance_scale: 0.0
-  chunk_size: 1
-  scheduling_matrix: autoregressive
-  noise_level: random_all
-  causal: true
-  diffusion:
-    objective: pred_v
-    beta_schedule: sigmoid
-    schedule_fn_kwargs: {}
-    clip_noise: 20.0
-    use_snr: false
-    use_cum_snr: false
-    use_fused_snr: true
-    snr_clip: 5.0
-    cum_snr_decay: 0.96
-    timesteps: 1000
-    sampling_timesteps: 20
-    ddim_sampling_eta: 0.0
-    stabilization_level: 15
-    architecture:
-      network_size: 64
-      attn_heads: 4
-      attn_dim_head: 64
-      dim_mults:
-      - 1
-      - 2
-      - 4
-      - 8
-      resolution: ${dataset.resolution}
-      attn_resolutions:
-      - 16
-      - 32
-      - 64
-      - 128
-      use_init_temporal_attn: true
-      use_linear_attn: true
-      time_emb_type: rotary
-  n_frames: ${dataset.n_frames}
-  metadata: ${dataset.metadata}
-  action_cond_dim: 25
-  use_plucker: true
-  _name: dememwm_memory_dit
-  memory_token_cross_attention: true
-  memory_cross_attn_layers:
-  - 3
-  - 7
-  - 11
-  - 15
-  pose_cond_dim: 5
-  log_video: true
-  dememwm:
-    enabled: true
-    training_stage: stage_1
-    debug_force_all_streams: false
-    curriculum:
-      enabled: true
-      full_stage_start_step: 20000
-      freeze_vae: true
-      dit_freeze:
-        enabled: true
-      lr:
-        dememwm_modules: 4.0e-05
-        memory_adapters: 4.0e-05
-        full_dit: 1.0e-05
-    spatial_kernel_size: 3
-    anchor:
-      enabled: true
-      anchor_indices:
-      - 0
-      - 1
-      - 2
-      - 3
-      allow_generated_as_anchor: false
-      diverse_selection: true
-      compress:
-        downsample_ratio: 2
-    dynamic:
-      enabled: true
-      recent_frames: 4
-      conv_kernel_t: 3
-      conv_stride_t: 2
-      compress:
-        downsample_ratio: 2
-    revisit:
-      enabled: true
-      deterministic_pose_retrieval: true
-      fov_overlap_threshold: 0.6
-      plucker_weight: 0.1
-      max_frames: 2
-      fov_half_h: 52.5
-      fov_half_v: 37.5
-      fov_radius: 30.0
-      fov_yaw_samples: 25
-      fov_pitch_samples: 20
-      fov_depth_samples: 20
-      pose_preselect_topk: 64
-      plucker_grid_h: 4
-      plucker_grid_w: 4
-      plucker_focal_length: 0.35
-      compress:
-        downsample_ratio: 2
-    stage_policy:
-      noise_bucket_logging: true
-    eval_ablation:
-      enabled: false
-      branch: A_plus_D_plus_R_normal
-    generated_history_proxy:
-      enabled: true
-      start_step: 40000
-      ramp_steps: 40000
-      max_prob: 0.25
-      noise_std: 0.25
-      dropout_prob: 0.0
-    injection:
-      memory_hidden_size: 512
-    cache:
-      enabled: true
-      device: cpu
-      keep_raw_latents: all
-      keep_compressed_records: true
-      keep_prefix_anchors: true
-      eviction_policy: none
-      no_evict: true
-      clear_between_videos: true
-      max_records: null
-      on_capacity_exceeded: warn
-    checkpoint:
-      strict_dememwm_eval_load: true
-  n_tokens: 4
-debug: false
-wandb:
-  entity: turlin
-  project: worldmem
-  mode: offline
-resume: null
-load: null
-auto_resume: true
-resume_ckpt_path: null
-name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-output_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-customized_load: true
-seperate_load: true
-diffusion_model_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors
-vae_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors
-only_tune_memory: false

outputs/2026-06-01/15-41-06/.hydra/hydra.yaml

@@ -1,232 +0,0 @@
-hydra:
-  run:
-    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
-  sweep:
-    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
-    subdir: ${hydra.job.num}
-  launcher:
-    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
-  sweeper:
-    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
-    max_batch_size: null
-    params: null
-  help:
-    app_name: ${hydra.job.name}
-    header: '${hydra.help.app_name} is powered by Hydra.
-
-      '
-    footer: 'Powered by Hydra (https://hydra.cc)
-
-      Use --hydra-help to view Hydra specific help
-
-      '
-    template: '${hydra.help.header}
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (group=option)
-
-
-      $APP_CONFIG_GROUPS
-
-
-      == Config ==
-
-      Override anything in the config (foo.bar=value)
-
-
-      $CONFIG
-
-
-      ${hydra.help.footer}
-
-      '
-  hydra_help:
-    template: 'Hydra (${hydra.runtime.version})
-
-      See https://hydra.cc for more info.
-
-
-      == Flags ==
-
-      $FLAGS_HELP
-
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
-      to command line)
-
-
-      $HYDRA_CONFIG_GROUPS
-
-
-      Use ''--cfg hydra'' to Show the Hydra config.
-
-      '
-    hydra_help: ???
-  hydra_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][HYDRA] %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-    root:
-      level: INFO
-      handlers:
-      - console
-    loggers:
-      logging_example:
-        level: DEBUG
-    disable_existing_loggers: false
-  job_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-      file:
-        class: logging.FileHandler
-        formatter: simple
-        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
-    root:
-      level: INFO
-      handlers:
-      - console
-      - file
-    disable_existing_loggers: false
-  env: {}
-  mode: RUN
-  searchpath: []
-  callbacks: {}
-  output_subdir: .hydra
-  overrides:
-    hydra:
-    - hydra.mode=RUN
-    task:
-    - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-    - +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-    - auto_resume=true
-    - experiment.tasks=[training]
-    - algorithm=dememwm_memory_dit
-    - +customized_load=true
-    - +seperate_load=true
-    - +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors
-    - +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors
-    - +only_tune_memory=false
-    - dataset=video_minecraft_latent
-    - dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft
-    - dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
-    - dataset.n_frames=1000
-    - +dataset.n_frames_valid=1100
-    - +dataset.customized_validation=true
-    - +dataset.wo_updown=false
-    - +dataset.angle_range=180
-    - +dataset.pos_range=8
-    - ++algorithm.n_tokens=4
-    - algorithm.x_shape=[16,18,32]
-    - ++algorithm.context_frames=100
-    - ++algorithm.log_video=true
-    - ++algorithm.diffusion.sampling_timesteps=20
-    - ++algorithm.dememwm.debug_force_all_streams=false
-    - ++algorithm.dememwm.generated_history_proxy.enabled=true
-    - ++algorithm.dememwm.generated_history_proxy.start_step=40000
-    - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-    - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-    - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-    - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-    - ++algorithm.dememwm.anchor.enabled=true
-    - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-    - ++algorithm.dememwm.anchor.diverse_selection=true
-    - ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-    - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-    - ++algorithm.dememwm.dynamic.enabled=true
-    - ++algorithm.dememwm.dynamic.recent_frames=4
-    - ++algorithm.dememwm.dynamic.conv_stride_t=2
-    - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-    - ++algorithm.dememwm.revisit.enabled=true
-    - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-    - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-    - ++algorithm.dememwm.revisit.pose_preselect_topk=64
-    - ++algorithm.dememwm.revisit.fov_yaw_samples=25
-    - ++algorithm.dememwm.revisit.fov_pitch_samples=20
-    - ++algorithm.dememwm.revisit.fov_depth_samples=20
-    - ++algorithm.dememwm.revisit.plucker_weight=0.10
-    - ++algorithm.dememwm.revisit.max_frames=2
-    - ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-    - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-    - ++algorithm.dememwm.cache.enabled=true
-    - ++algorithm.dememwm.cache.device=cpu
-    - ++algorithm.dememwm.cache.keep_raw_latents=all
-    - ++algorithm.dememwm.cache.keep_compressed_records=true
-    - ++algorithm.dememwm.cache.eviction_policy=none
-    - ++algorithm.dememwm.cache.no_evict=true
-    - ++algorithm.dememwm.cache.clear_between_videos=true
-    - ++algorithm.dememwm.cache.max_records=null
-    - ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-    - ++algorithm.dememwm.curriculum.enabled=true
-    - ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-    - ++algorithm.dememwm.curriculum.freeze_vae=true
-    - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-    - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-    - experiment.training.batch_size=16
-    - experiment.training.optim.accumulate_grad_batches=1
-    - experiment.validation.batch_size=1
-    - experiment.validation.limit_batch=2
-    - experiment.training.checkpointing.every_n_train_steps=2000
-    - experiment.validation.val_every_n_step=2000
-    - experiment.training.max_steps=350000
-  job:
-    name: main
-    chdir: null
-    override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features,dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000
-    id: ???
-    num: ???
-    config_name: training
-    env_set: {}
-    env_copy: []
-    config:
-      override_dirname:
-        kv_sep: '='
-        item_sep: ','
-        exclude_keys: []
-  runtime:
-    version: 1.3.2
-    version_base: '1.3'
-    cwd: /proj/cvl/users/x_fahkh2/DeMemWM
-    config_sources:
-    - path: hydra.conf
-      schema: pkg
-      provider: hydra
-    - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations
-      schema: file
-      provider: main
-    - path: ''
-      schema: structured
-      provider: schema
-    output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-01/15-41-06
-    choices:
-      cluster: null
-      algorithm: dememwm_memory_dit
-      dataset: video_minecraft_latent
-      experiment: exp_video
-      hydra/env: default
-      hydra/callbacks: null
-      hydra/job_logging: default
-      hydra/hydra_logging: default
-      hydra/hydra_help: default
-      hydra/help: default
-      hydra/sweeper: basic
-      hydra/launcher: basic
-      hydra/output: default
-  verbose: false

outputs/2026-06-01/15-41-06/.hydra/overrides.yaml

@@ -1,74 +0,0 @@
-- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-- +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-- auto_resume=true
-- experiment.tasks=[training]
-- algorithm=dememwm_memory_dit
-- +customized_load=true
-- +seperate_load=true
-- +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors
-- +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors
-- +only_tune_memory=false
-- dataset=video_minecraft_latent
-- dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft
-- dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
-- dataset.n_frames=1000
-- +dataset.n_frames_valid=1100
-- +dataset.customized_validation=true
-- +dataset.wo_updown=false
-- +dataset.angle_range=180
-- +dataset.pos_range=8
-- ++algorithm.n_tokens=4
-- algorithm.x_shape=[16,18,32]
-- ++algorithm.context_frames=100
-- ++algorithm.log_video=true
-- ++algorithm.diffusion.sampling_timesteps=20
-- ++algorithm.dememwm.debug_force_all_streams=false
-- ++algorithm.dememwm.generated_history_proxy.enabled=true
-- ++algorithm.dememwm.generated_history_proxy.start_step=40000
-- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-- ++algorithm.dememwm.anchor.enabled=true
-- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-- ++algorithm.dememwm.anchor.diverse_selection=true
-- ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-- ++algorithm.dememwm.dynamic.enabled=true
-- ++algorithm.dememwm.dynamic.recent_frames=4
-- ++algorithm.dememwm.dynamic.conv_stride_t=2
-- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-- ++algorithm.dememwm.revisit.enabled=true
-- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-- ++algorithm.dememwm.revisit.pose_preselect_topk=64
-- ++algorithm.dememwm.revisit.fov_yaw_samples=25
-- ++algorithm.dememwm.revisit.fov_pitch_samples=20
-- ++algorithm.dememwm.revisit.fov_depth_samples=20
-- ++algorithm.dememwm.revisit.plucker_weight=0.10
-- ++algorithm.dememwm.revisit.max_frames=2
-- ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-- ++algorithm.dememwm.cache.enabled=true
-- ++algorithm.dememwm.cache.device=cpu
-- ++algorithm.dememwm.cache.keep_raw_latents=all
-- ++algorithm.dememwm.cache.keep_compressed_records=true
-- ++algorithm.dememwm.cache.eviction_policy=none
-- ++algorithm.dememwm.cache.no_evict=true
-- ++algorithm.dememwm.cache.clear_between_videos=true
-- ++algorithm.dememwm.cache.max_records=null
-- ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-- ++algorithm.dememwm.curriculum.enabled=true
-- ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-- ++algorithm.dememwm.curriculum.freeze_vae=true
-- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-- experiment.training.batch_size=16
-- experiment.training.optim.accumulate_grad_batches=1
-- experiment.validation.batch_size=1
-- experiment.validation.limit_batch=2
-- experiment.training.checkpointing.every_n_train_steps=2000
-- experiment.validation.val_every_n_step=2000
-- experiment.training.max_steps=350000

outputs/2026-06-01/15-41-07/.hydra/config.yaml

@@ -1,234 +0,0 @@
-experiment:
-  debug: ${debug}
-  tasks:
-  - training
-  num_nodes: 1
-  training:
-    precision: 16-mixed
-    compile: false
-    lr: 2.0e-05
-    batch_size: 16
-    max_epochs: -1
-    max_steps: 350000
-    max_time: null
-    data:
-      num_workers: 4
-      shuffle: true
-    optim:
-      accumulate_grad_batches: 1
-      gradient_clip_val: 1.0
-    checkpointing:
-      every_n_train_steps: 2000
-      every_n_epochs: null
-      train_time_interval: null
-      save_last: true
-      enable_version_counter: false
-  validation:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    val_every_n_step: 2000
-    val_every_n_epoch: null
-    limit_batch: 2
-    inference_mode: true
-    data:
-      num_workers: 4
-      shuffle: false
-  test:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    limit_batch: 1
-    data:
-      num_workers: 4
-      shuffle: false
-  logging:
-    metrics: null
-dataset:
-  debug: ${debug}
-  metadata: data/${dataset.name}/metadata.json
-  data_mean: 0.5
-  data_std: 0.5
-  save_dir: /share_1/users/bonan_ding/worldmem_data/minecraft
-  n_frames: 1000
-  context_length: 1
-  resolution: 128
-  observation_shape:
-  - 3
-  - ${dataset.resolution}
-  - ${dataset.resolution}
-  external_cond_dim: 0
-  validation_multiplier: 1
-  frame_skip: 1
-  action_cond_dim: 25
-  _name: video_minecraft_latent
-  precomputed_feature_dir: /share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
-  n_frames_valid: 1100
-  customized_validation: true
-  wo_updown: false
-  angle_range: 180
-  pos_range: 8
-algorithm:
-  debug: ${debug}
-  lr: ${experiment.training.lr}
-  x_shape:
-  - 16
-  - 18
-  - 32
-  frame_stack: 1
-  frame_skip: ${dataset.frame_skip}
-  data_mean: ${dataset.data_mean}
-  data_std: ${dataset.data_std}
-  external_cond_dim: 0
-  context_frames: 100
-  weight_decay: 0.002
-  warmup_steps: 1000
-  optimizer_beta:
-  - 0.9
-  - 0.99
-  uncertainty_scale: 1
-  guidance_scale: 0.0
-  chunk_size: 1
-  scheduling_matrix: autoregressive
-  noise_level: random_all
-  causal: true
-  diffusion:
-    objective: pred_v
-    beta_schedule: sigmoid
-    schedule_fn_kwargs: {}
-    clip_noise: 20.0
-    use_snr: false
-    use_cum_snr: false
-    use_fused_snr: true
-    snr_clip: 5.0
-    cum_snr_decay: 0.96
-    timesteps: 1000
-    sampling_timesteps: 20
-    ddim_sampling_eta: 0.0
-    stabilization_level: 15
-    architecture:
-      network_size: 64
-      attn_heads: 4
-      attn_dim_head: 64
-      dim_mults:
-      - 1
-      - 2
-      - 4
-      - 8
-      resolution: ${dataset.resolution}
-      attn_resolutions:
-      - 16
-      - 32
-      - 64
-      - 128
-      use_init_temporal_attn: true
-      use_linear_attn: true
-      time_emb_type: rotary
-  n_frames: ${dataset.n_frames}
-  metadata: ${dataset.metadata}
-  action_cond_dim: 25
-  use_plucker: true
-  _name: dememwm_memory_dit
-  memory_token_cross_attention: true
-  memory_cross_attn_layers:
-  - 3
-  - 7
-  - 11
-  - 15
-  pose_cond_dim: 5
-  log_video: true
-  dememwm:
-    enabled: true
-    training_stage: stage_1
-    debug_force_all_streams: false
-    curriculum:
-      enabled: true
-      full_stage_start_step: 20000
-      freeze_vae: true
-      dit_freeze:
-        enabled: true
-      lr:
-        dememwm_modules: 4.0e-05
-        memory_adapters: 4.0e-05
-        full_dit: 1.0e-05
-    spatial_kernel_size: 3
-    anchor:
-      enabled: true
-      anchor_indices:
-      - 0
-      - 1
-      - 2
-      - 3
-      allow_generated_as_anchor: false
-      diverse_selection: true
-      compress:
-        downsample_ratio: 2
-    dynamic:
-      enabled: true
-      recent_frames: 4
-      conv_kernel_t: 3
-      conv_stride_t: 2
-      compress:
-        downsample_ratio: 2
-    revisit:
-      enabled: true
-      deterministic_pose_retrieval: true
-      fov_overlap_threshold: 0.6
-      plucker_weight: 0.1
-      max_frames: 2
-      fov_half_h: 52.5
-      fov_half_v: 37.5
-      fov_radius: 30.0
-      fov_yaw_samples: 25
-      fov_pitch_samples: 20
-      fov_depth_samples: 20
-      pose_preselect_topk: 64
-      plucker_grid_h: 4
-      plucker_grid_w: 4
-      plucker_focal_length: 0.35
-      compress:
-        downsample_ratio: 2
-    stage_policy:
-      noise_bucket_logging: true
-    eval_ablation:
-      enabled: false
-      branch: A_plus_D_plus_R_normal
-    generated_history_proxy:
-      enabled: true
-      start_step: 40000
-      ramp_steps: 40000
-      max_prob: 0.25
-      noise_std: 0.25
-      dropout_prob: 0.0
-    injection:
-      memory_hidden_size: 512
-    cache:
-      enabled: true
-      device: cpu
-      keep_raw_latents: all
-      keep_compressed_records: true
-      keep_prefix_anchors: true
-      eviction_policy: none
-      no_evict: true
-      clear_between_videos: true
-      max_records: null
-      on_capacity_exceeded: warn
-    checkpoint:
-      strict_dememwm_eval_load: true
-  n_tokens: 4
-debug: false
-wandb:
-  entity: turlin
-  project: worldmem
-  mode: offline
-resume: null
-load: null
-auto_resume: true
-resume_ckpt_path: null
-name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-output_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-customized_load: true
-seperate_load: true
-diffusion_model_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors
-vae_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors
-only_tune_memory: false

outputs/2026-06-01/15-41-07/.hydra/hydra.yaml

@@ -1,232 +0,0 @@
-hydra:
-  run:
-    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
-  sweep:
-    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
-    subdir: ${hydra.job.num}
-  launcher:
-    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
-  sweeper:
-    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
-    max_batch_size: null
-    params: null
-  help:
-    app_name: ${hydra.job.name}
-    header: '${hydra.help.app_name} is powered by Hydra.
-
-      '
-    footer: 'Powered by Hydra (https://hydra.cc)
-
-      Use --hydra-help to view Hydra specific help
-
-      '
-    template: '${hydra.help.header}
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (group=option)
-
-
-      $APP_CONFIG_GROUPS
-
-
-      == Config ==
-
-      Override anything in the config (foo.bar=value)
-
-
-      $CONFIG
-
-
-      ${hydra.help.footer}
-
-      '
-  hydra_help:
-    template: 'Hydra (${hydra.runtime.version})
-
-      See https://hydra.cc for more info.
-
-
-      == Flags ==
-
-      $FLAGS_HELP
-
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
-      to command line)
-
-
-      $HYDRA_CONFIG_GROUPS
-
-
-      Use ''--cfg hydra'' to Show the Hydra config.
-
-      '
-    hydra_help: ???
-  hydra_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][HYDRA] %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-    root:
-      level: INFO
-      handlers:
-      - console
-    loggers:
-      logging_example:
-        level: DEBUG
-    disable_existing_loggers: false
-  job_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-      file:
-        class: logging.FileHandler
-        formatter: simple
-        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
-    root:
-      level: INFO
-      handlers:
-      - console
-      - file
-    disable_existing_loggers: false
-  env: {}
-  mode: RUN
-  searchpath: []
-  callbacks: {}
-  output_subdir: .hydra
-  overrides:
-    hydra:
-    - hydra.mode=RUN
-    task:
-    - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-    - +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-    - auto_resume=true
-    - experiment.tasks=[training]
-    - algorithm=dememwm_memory_dit
-    - +customized_load=true
-    - +seperate_load=true
-    - +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors
-    - +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors
-    - +only_tune_memory=false
-    - dataset=video_minecraft_latent
-    - dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft
-    - dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
-    - dataset.n_frames=1000
-    - +dataset.n_frames_valid=1100
-    - +dataset.customized_validation=true
-    - +dataset.wo_updown=false
-    - +dataset.angle_range=180
-    - +dataset.pos_range=8
-    - ++algorithm.n_tokens=4
-    - algorithm.x_shape=[16,18,32]
-    - ++algorithm.context_frames=100
-    - ++algorithm.log_video=true
-    - ++algorithm.diffusion.sampling_timesteps=20
-    - ++algorithm.dememwm.debug_force_all_streams=false
-    - ++algorithm.dememwm.generated_history_proxy.enabled=true
-    - ++algorithm.dememwm.generated_history_proxy.start_step=40000
-    - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-    - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-    - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-    - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-    - ++algorithm.dememwm.anchor.enabled=true
-    - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-    - ++algorithm.dememwm.anchor.diverse_selection=true
-    - ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-    - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-    - ++algorithm.dememwm.dynamic.enabled=true
-    - ++algorithm.dememwm.dynamic.recent_frames=4
-    - ++algorithm.dememwm.dynamic.conv_stride_t=2
-    - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-    - ++algorithm.dememwm.revisit.enabled=true
-    - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-    - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-    - ++algorithm.dememwm.revisit.pose_preselect_topk=64
-    - ++algorithm.dememwm.revisit.fov_yaw_samples=25
-    - ++algorithm.dememwm.revisit.fov_pitch_samples=20
-    - ++algorithm.dememwm.revisit.fov_depth_samples=20
-    - ++algorithm.dememwm.revisit.plucker_weight=0.10
-    - ++algorithm.dememwm.revisit.max_frames=2
-    - ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-    - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-    - ++algorithm.dememwm.cache.enabled=true
-    - ++algorithm.dememwm.cache.device=cpu
-    - ++algorithm.dememwm.cache.keep_raw_latents=all
-    - ++algorithm.dememwm.cache.keep_compressed_records=true
-    - ++algorithm.dememwm.cache.eviction_policy=none
-    - ++algorithm.dememwm.cache.no_evict=true
-    - ++algorithm.dememwm.cache.clear_between_videos=true
-    - ++algorithm.dememwm.cache.max_records=null
-    - ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-    - ++algorithm.dememwm.curriculum.enabled=true
-    - ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-    - ++algorithm.dememwm.curriculum.freeze_vae=true
-    - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-    - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-    - experiment.training.batch_size=16
-    - experiment.training.optim.accumulate_grad_batches=1
-    - experiment.validation.batch_size=1
-    - experiment.validation.limit_batch=2
-    - experiment.training.checkpointing.every_n_train_steps=2000
-    - experiment.validation.val_every_n_step=2000
-    - experiment.training.max_steps=350000
-  job:
-    name: main
-    chdir: null
-    override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features,dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000
-    id: ???
-    num: ???
-    config_name: training
-    env_set: {}
-    env_copy: []
-    config:
-      override_dirname:
-        kv_sep: '='
-        item_sep: ','
-        exclude_keys: []
-  runtime:
-    version: 1.3.2
-    version_base: '1.3'
-    cwd: /proj/cvl/users/x_fahkh2/DeMemWM
-    config_sources:
-    - path: hydra.conf
-      schema: pkg
-      provider: hydra
-    - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations
-      schema: file
-      provider: main
-    - path: ''
-      schema: structured
-      provider: schema
-    output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-01/15-41-07
-    choices:
-      cluster: null
-      algorithm: dememwm_memory_dit
-      dataset: video_minecraft_latent
-      experiment: exp_video
-      hydra/env: default
-      hydra/callbacks: null
-      hydra/job_logging: default
-      hydra/hydra_logging: default
-      hydra/hydra_help: default
-      hydra/help: default
-      hydra/sweeper: basic
-      hydra/launcher: basic
-      hydra/output: default
-  verbose: false

outputs/2026-06-01/15-41-07/.hydra/overrides.yaml

@@ -1,74 +0,0 @@
-- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-- +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-- auto_resume=true
-- experiment.tasks=[training]
-- algorithm=dememwm_memory_dit
-- +customized_load=true
-- +seperate_load=true
-- +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors
-- +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors
-- +only_tune_memory=false
-- dataset=video_minecraft_latent
-- dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft
-- dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
-- dataset.n_frames=1000
-- +dataset.n_frames_valid=1100
-- +dataset.customized_validation=true
-- +dataset.wo_updown=false
-- +dataset.angle_range=180
-- +dataset.pos_range=8
-- ++algorithm.n_tokens=4
-- algorithm.x_shape=[16,18,32]
-- ++algorithm.context_frames=100
-- ++algorithm.log_video=true
-- ++algorithm.diffusion.sampling_timesteps=20
-- ++algorithm.dememwm.debug_force_all_streams=false
-- ++algorithm.dememwm.generated_history_proxy.enabled=true
-- ++algorithm.dememwm.generated_history_proxy.start_step=40000
-- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-- ++algorithm.dememwm.anchor.enabled=true
-- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-- ++algorithm.dememwm.anchor.diverse_selection=true
-- ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-- ++algorithm.dememwm.dynamic.enabled=true
-- ++algorithm.dememwm.dynamic.recent_frames=4
-- ++algorithm.dememwm.dynamic.conv_stride_t=2
-- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-- ++algorithm.dememwm.revisit.enabled=true
-- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-- ++algorithm.dememwm.revisit.pose_preselect_topk=64
-- ++algorithm.dememwm.revisit.fov_yaw_samples=25
-- ++algorithm.dememwm.revisit.fov_pitch_samples=20
-- ++algorithm.dememwm.revisit.fov_depth_samples=20
-- ++algorithm.dememwm.revisit.plucker_weight=0.10
-- ++algorithm.dememwm.revisit.max_frames=2
-- ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-- ++algorithm.dememwm.cache.enabled=true
-- ++algorithm.dememwm.cache.device=cpu
-- ++algorithm.dememwm.cache.keep_raw_latents=all
-- ++algorithm.dememwm.cache.keep_compressed_records=true
-- ++algorithm.dememwm.cache.eviction_policy=none
-- ++algorithm.dememwm.cache.no_evict=true
-- ++algorithm.dememwm.cache.clear_between_videos=true
-- ++algorithm.dememwm.cache.max_records=null
-- ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-- ++algorithm.dememwm.curriculum.enabled=true
-- ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-- ++algorithm.dememwm.curriculum.freeze_vae=true
-- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-- experiment.training.batch_size=16
-- experiment.training.optim.accumulate_grad_batches=1
-- experiment.validation.batch_size=1
-- experiment.validation.limit_batch=2
-- experiment.training.checkpointing.every_n_train_steps=2000
-- experiment.validation.val_every_n_step=2000
-- experiment.training.max_steps=350000

outputs/2026-06-01/15-41-08/.hydra/config.yaml

@@ -1,234 +0,0 @@
-experiment:
-  debug: ${debug}
-  tasks:
-  - training
-  num_nodes: 1
-  training:
-    precision: 16-mixed
-    compile: false
-    lr: 2.0e-05
-    batch_size: 16
-    max_epochs: -1
-    max_steps: 350000
-    max_time: null
-    data:
-      num_workers: 4
-      shuffle: true
-    optim:
-      accumulate_grad_batches: 1
-      gradient_clip_val: 1.0
-    checkpointing:
-      every_n_train_steps: 2000
-      every_n_epochs: null
-      train_time_interval: null
-      save_last: true
-      enable_version_counter: false
-  validation:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    val_every_n_step: 2000
-    val_every_n_epoch: null
-    limit_batch: 2
-    inference_mode: true
-    data:
-      num_workers: 4
-      shuffle: false
-  test:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    limit_batch: 1
-    data:
-      num_workers: 4
-      shuffle: false
-  logging:
-    metrics: null
-dataset:
-  debug: ${debug}
-  metadata: data/${dataset.name}/metadata.json
-  data_mean: 0.5
-  data_std: 0.5
-  save_dir: /share_1/users/bonan_ding/worldmem_data/minecraft
-  n_frames: 1000
-  context_length: 1
-  resolution: 128
-  observation_shape:
-  - 3
-  - ${dataset.resolution}
-  - ${dataset.resolution}
-  external_cond_dim: 0
-  validation_multiplier: 1
-  frame_skip: 1
-  action_cond_dim: 25
-  _name: video_minecraft_latent
-  precomputed_feature_dir: /share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
-  n_frames_valid: 1100
-  customized_validation: true
-  wo_updown: false
-  angle_range: 180
-  pos_range: 8
-algorithm:
-  debug: ${debug}
-  lr: ${experiment.training.lr}
-  x_shape:
-  - 16
-  - 18
-  - 32
-  frame_stack: 1
-  frame_skip: ${dataset.frame_skip}
-  data_mean: ${dataset.data_mean}
-  data_std: ${dataset.data_std}
-  external_cond_dim: 0
-  context_frames: 100
-  weight_decay: 0.002
-  warmup_steps: 1000
-  optimizer_beta:
-  - 0.9
-  - 0.99
-  uncertainty_scale: 1
-  guidance_scale: 0.0
-  chunk_size: 1
-  scheduling_matrix: autoregressive
-  noise_level: random_all
-  causal: true
-  diffusion:
-    objective: pred_v
-    beta_schedule: sigmoid
-    schedule_fn_kwargs: {}
-    clip_noise: 20.0
-    use_snr: false
-    use_cum_snr: false
-    use_fused_snr: true
-    snr_clip: 5.0
-    cum_snr_decay: 0.96
-    timesteps: 1000
-    sampling_timesteps: 20
-    ddim_sampling_eta: 0.0
-    stabilization_level: 15
-    architecture:
-      network_size: 64
-      attn_heads: 4
-      attn_dim_head: 64
-      dim_mults:
-      - 1
-      - 2
-      - 4
-      - 8
-      resolution: ${dataset.resolution}
-      attn_resolutions:
-      - 16
-      - 32
-      - 64
-      - 128
-      use_init_temporal_attn: true
-      use_linear_attn: true
-      time_emb_type: rotary
-  n_frames: ${dataset.n_frames}
-  metadata: ${dataset.metadata}
-  action_cond_dim: 25
-  use_plucker: true
-  _name: dememwm_memory_dit
-  memory_token_cross_attention: true
-  memory_cross_attn_layers:
-  - 3
-  - 7
-  - 11
-  - 15
-  pose_cond_dim: 5
-  log_video: true
-  dememwm:
-    enabled: true
-    training_stage: stage_1
-    debug_force_all_streams: false
-    curriculum:
-      enabled: true
-      full_stage_start_step: 20000
-      freeze_vae: true
-      dit_freeze:
-        enabled: true
-      lr:
-        dememwm_modules: 4.0e-05
-        memory_adapters: 4.0e-05
-        full_dit: 1.0e-05
-    spatial_kernel_size: 3
-    anchor:
-      enabled: true
-      anchor_indices:
-      - 0
-      - 1
-      - 2
-      - 3
-      allow_generated_as_anchor: false
-      diverse_selection: true
-      compress:
-        downsample_ratio: 2
-    dynamic:
-      enabled: true
-      recent_frames: 4
-      conv_kernel_t: 3
-      conv_stride_t: 2
-      compress:
-        downsample_ratio: 2
-    revisit:
-      enabled: true
-      deterministic_pose_retrieval: true
-      fov_overlap_threshold: 0.6
-      plucker_weight: 0.1
-      max_frames: 2
-      fov_half_h: 52.5
-      fov_half_v: 37.5
-      fov_radius: 30.0
-      fov_yaw_samples: 25
-      fov_pitch_samples: 20
-      fov_depth_samples: 20
-      pose_preselect_topk: 64
-      plucker_grid_h: 4
-      plucker_grid_w: 4
-      plucker_focal_length: 0.35
-      compress:
-        downsample_ratio: 2
-    stage_policy:
-      noise_bucket_logging: true
-    eval_ablation:
-      enabled: false
-      branch: A_plus_D_plus_R_normal
-    generated_history_proxy:
-      enabled: true
-      start_step: 40000
-      ramp_steps: 40000
-      max_prob: 0.25
-      noise_std: 0.25
-      dropout_prob: 0.0
-    injection:
-      memory_hidden_size: 512
-    cache:
-      enabled: true
-      device: cpu
-      keep_raw_latents: all
-      keep_compressed_records: true
-      keep_prefix_anchors: true
-      eviction_policy: none
-      no_evict: true
-      clear_between_videos: true
-      max_records: null
-      on_capacity_exceeded: warn
-    checkpoint:
-      strict_dememwm_eval_load: true
-  n_tokens: 4
-debug: false
-wandb:
-  entity: turlin
-  project: worldmem
-  mode: offline
-resume: null
-load: null
-auto_resume: true
-resume_ckpt_path: null
-name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-output_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-customized_load: true
-seperate_load: true
-diffusion_model_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors
-vae_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors
-only_tune_memory: false

outputs/2026-06-01/15-41-08/.hydra/hydra.yaml

@@ -1,232 +0,0 @@
-hydra:
-  run:
-    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
-  sweep:
-    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
-    subdir: ${hydra.job.num}
-  launcher:
-    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
-  sweeper:
-    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
-    max_batch_size: null
-    params: null
-  help:
-    app_name: ${hydra.job.name}
-    header: '${hydra.help.app_name} is powered by Hydra.
-
-      '
-    footer: 'Powered by Hydra (https://hydra.cc)
-
-      Use --hydra-help to view Hydra specific help
-
-      '
-    template: '${hydra.help.header}
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (group=option)
-
-
-      $APP_CONFIG_GROUPS
-
-
-      == Config ==
-
-      Override anything in the config (foo.bar=value)
-
-
-      $CONFIG
-
-
-      ${hydra.help.footer}
-
-      '
-  hydra_help:
-    template: 'Hydra (${hydra.runtime.version})
-
-      See https://hydra.cc for more info.
-
-
-      == Flags ==
-
-      $FLAGS_HELP
-
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
-      to command line)
-
-
-      $HYDRA_CONFIG_GROUPS
-
-
-      Use ''--cfg hydra'' to Show the Hydra config.
-
-      '
-    hydra_help: ???
-  hydra_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][HYDRA] %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-    root:
-      level: INFO
-      handlers:
-      - console
-    loggers:
-      logging_example:
-        level: DEBUG
-    disable_existing_loggers: false
-  job_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-      file:
-        class: logging.FileHandler
-        formatter: simple
-        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
-    root:
-      level: INFO
-      handlers:
-      - console
-      - file
-    disable_existing_loggers: false
-  env: {}
-  mode: RUN
-  searchpath: []
-  callbacks: {}
-  output_subdir: .hydra
-  overrides:
-    hydra:
-    - hydra.mode=RUN
-    task:
-    - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-    - +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-    - auto_resume=true
-    - experiment.tasks=[training]
-    - algorithm=dememwm_memory_dit
-    - +customized_load=true
-    - +seperate_load=true
-    - +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors
-    - +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors
-    - +only_tune_memory=false
-    - dataset=video_minecraft_latent
-    - dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft
-    - dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
-    - dataset.n_frames=1000
-    - +dataset.n_frames_valid=1100
-    - +dataset.customized_validation=true
-    - +dataset.wo_updown=false
-    - +dataset.angle_range=180
-    - +dataset.pos_range=8
-    - ++algorithm.n_tokens=4
-    - algorithm.x_shape=[16,18,32]
-    - ++algorithm.context_frames=100
-    - ++algorithm.log_video=true
-    - ++algorithm.diffusion.sampling_timesteps=20
-    - ++algorithm.dememwm.debug_force_all_streams=false
-    - ++algorithm.dememwm.generated_history_proxy.enabled=true
-    - ++algorithm.dememwm.generated_history_proxy.start_step=40000
-    - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-    - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-    - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-    - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-    - ++algorithm.dememwm.anchor.enabled=true
-    - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-    - ++algorithm.dememwm.anchor.diverse_selection=true
-    - ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-    - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-    - ++algorithm.dememwm.dynamic.enabled=true
-    - ++algorithm.dememwm.dynamic.recent_frames=4
-    - ++algorithm.dememwm.dynamic.conv_stride_t=2
-    - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-    - ++algorithm.dememwm.revisit.enabled=true
-    - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-    - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-    - ++algorithm.dememwm.revisit.pose_preselect_topk=64
-    - ++algorithm.dememwm.revisit.fov_yaw_samples=25
-    - ++algorithm.dememwm.revisit.fov_pitch_samples=20
-    - ++algorithm.dememwm.revisit.fov_depth_samples=20
-    - ++algorithm.dememwm.revisit.plucker_weight=0.10
-    - ++algorithm.dememwm.revisit.max_frames=2
-    - ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-    - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-    - ++algorithm.dememwm.cache.enabled=true
-    - ++algorithm.dememwm.cache.device=cpu
-    - ++algorithm.dememwm.cache.keep_raw_latents=all
-    - ++algorithm.dememwm.cache.keep_compressed_records=true
-    - ++algorithm.dememwm.cache.eviction_policy=none
-    - ++algorithm.dememwm.cache.no_evict=true
-    - ++algorithm.dememwm.cache.clear_between_videos=true
-    - ++algorithm.dememwm.cache.max_records=null
-    - ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-    - ++algorithm.dememwm.curriculum.enabled=true
-    - ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-    - ++algorithm.dememwm.curriculum.freeze_vae=true
-    - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-    - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-    - experiment.training.batch_size=16
-    - experiment.training.optim.accumulate_grad_batches=1
-    - experiment.validation.batch_size=1
-    - experiment.validation.limit_batch=2
-    - experiment.training.checkpointing.every_n_train_steps=2000
-    - experiment.validation.val_every_n_step=2000
-    - experiment.training.max_steps=350000
-  job:
-    name: main
-    chdir: null
-    override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features,dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000
-    id: ???
-    num: ???
-    config_name: training
-    env_set: {}
-    env_copy: []
-    config:
-      override_dirname:
-        kv_sep: '='
-        item_sep: ','
-        exclude_keys: []
-  runtime:
-    version: 1.3.2
-    version_base: '1.3'
-    cwd: /proj/cvl/users/x_fahkh2/DeMemWM
-    config_sources:
-    - path: hydra.conf
-      schema: pkg
-      provider: hydra
-    - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations
-      schema: file
-      provider: main
-    - path: ''
-      schema: structured
-      provider: schema
-    output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-01/15-41-08
-    choices:
-      cluster: null
-      algorithm: dememwm_memory_dit
-      dataset: video_minecraft_latent
-      experiment: exp_video
-      hydra/env: default
-      hydra/callbacks: null
-      hydra/job_logging: default
-      hydra/hydra_logging: default
-      hydra/hydra_help: default
-      hydra/help: default
-      hydra/sweeper: basic
-      hydra/launcher: basic
-      hydra/output: default
-  verbose: false

outputs/2026-06-01/15-41-08/.hydra/overrides.yaml

@@ -1,74 +0,0 @@
-- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-- +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-- auto_resume=true
-- experiment.tasks=[training]
-- algorithm=dememwm_memory_dit
-- +customized_load=true
-- +seperate_load=true
-- +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors
-- +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors
-- +only_tune_memory=false
-- dataset=video_minecraft_latent
-- dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft
-- dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
-- dataset.n_frames=1000
-- +dataset.n_frames_valid=1100
-- +dataset.customized_validation=true
-- +dataset.wo_updown=false
-- +dataset.angle_range=180
-- +dataset.pos_range=8
-- ++algorithm.n_tokens=4
-- algorithm.x_shape=[16,18,32]
-- ++algorithm.context_frames=100
-- ++algorithm.log_video=true
-- ++algorithm.diffusion.sampling_timesteps=20
-- ++algorithm.dememwm.debug_force_all_streams=false
-- ++algorithm.dememwm.generated_history_proxy.enabled=true
-- ++algorithm.dememwm.generated_history_proxy.start_step=40000
-- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-- ++algorithm.dememwm.anchor.enabled=true
-- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-- ++algorithm.dememwm.anchor.diverse_selection=true
-- ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-- ++algorithm.dememwm.dynamic.enabled=true
-- ++algorithm.dememwm.dynamic.recent_frames=4
-- ++algorithm.dememwm.dynamic.conv_stride_t=2
-- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-- ++algorithm.dememwm.revisit.enabled=true
-- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-- ++algorithm.dememwm.revisit.pose_preselect_topk=64
-- ++algorithm.dememwm.revisit.fov_yaw_samples=25
-- ++algorithm.dememwm.revisit.fov_pitch_samples=20
-- ++algorithm.dememwm.revisit.fov_depth_samples=20
-- ++algorithm.dememwm.revisit.plucker_weight=0.10
-- ++algorithm.dememwm.revisit.max_frames=2
-- ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-- ++algorithm.dememwm.cache.enabled=true
-- ++algorithm.dememwm.cache.device=cpu
-- ++algorithm.dememwm.cache.keep_raw_latents=all
-- ++algorithm.dememwm.cache.keep_compressed_records=true
-- ++algorithm.dememwm.cache.eviction_policy=none
-- ++algorithm.dememwm.cache.no_evict=true
-- ++algorithm.dememwm.cache.clear_between_videos=true
-- ++algorithm.dememwm.cache.max_records=null
-- ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-- ++algorithm.dememwm.curriculum.enabled=true
-- ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-- ++algorithm.dememwm.curriculum.freeze_vae=true
-- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-- experiment.training.batch_size=16
-- experiment.training.optim.accumulate_grad_batches=1
-- experiment.validation.batch_size=1
-- experiment.validation.limit_batch=2
-- experiment.training.checkpointing.every_n_train_steps=2000
-- experiment.validation.val_every_n_step=2000
-- experiment.training.max_steps=350000

outputs/2026-06-01/15-41-08/main.log

@@ -1,5 +0,0 @@
-[2026-06-01 15:41:49,208][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP)
-[2026-06-01 15:41:49,334][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True
-[2026-06-01 15:41:49,334][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores
-[2026-06-01 15:41:49,334][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs
-[2026-06-01 15:41:49,334][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs

outputs/2026-06-03/20-22-54/.hydra/config.yaml

@@ -1,234 +0,0 @@
-experiment:
-  debug: ${debug}
-  tasks:
-  - training
-  num_nodes: 1
-  training:
-    precision: 16-mixed
-    compile: false
-    lr: 2.0e-05
-    batch_size: 16
-    max_epochs: -1
-    max_steps: 350000
-    max_time: null
-    data:
-      num_workers: 4
-      shuffle: true
-    optim:
-      accumulate_grad_batches: 1
-      gradient_clip_val: 1.0
-    checkpointing:
-      every_n_train_steps: 2000
-      every_n_epochs: null
-      train_time_interval: null
-      save_last: true
-      enable_version_counter: false
-  validation:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    val_every_n_step: 2000
-    val_every_n_epoch: null
-    limit_batch: 2
-    inference_mode: true
-    data:
-      num_workers: 4
-      shuffle: false
-  test:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    limit_batch: 1
-    data:
-      num_workers: 4
-      shuffle: false
-  logging:
-    metrics: null
-dataset:
-  debug: ${debug}
-  metadata: data/${dataset.name}/metadata.json
-  data_mean: 0.5
-  data_std: 0.5
-  save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-  n_frames: 1000
-  context_length: 1
-  resolution: 128
-  observation_shape:
-  - 3
-  - ${dataset.resolution}
-  - ${dataset.resolution}
-  external_cond_dim: 0
-  validation_multiplier: 1
-  frame_skip: 1
-  action_cond_dim: 25
-  _name: video_minecraft_latent
-  precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-  n_frames_valid: 1100
-  customized_validation: true
-  wo_updown: false
-  angle_range: 180
-  pos_range: 8
-algorithm:
-  debug: ${debug}
-  lr: ${experiment.training.lr}
-  x_shape:
-  - 16
-  - 18
-  - 32
-  frame_stack: 1
-  frame_skip: ${dataset.frame_skip}
-  data_mean: ${dataset.data_mean}
-  data_std: ${dataset.data_std}
-  external_cond_dim: 0
-  context_frames: 100
-  weight_decay: 0.002
-  warmup_steps: 1000
-  optimizer_beta:
-  - 0.9
-  - 0.99
-  uncertainty_scale: 1
-  guidance_scale: 0.0
-  chunk_size: 1
-  scheduling_matrix: autoregressive
-  noise_level: random_all
-  causal: true
-  diffusion:
-    objective: pred_v
-    beta_schedule: sigmoid
-    schedule_fn_kwargs: {}
-    clip_noise: 20.0
-    use_snr: false
-    use_cum_snr: false
-    use_fused_snr: true
-    snr_clip: 5.0
-    cum_snr_decay: 0.96
-    timesteps: 1000
-    sampling_timesteps: 20
-    ddim_sampling_eta: 0.0
-    stabilization_level: 15
-    architecture:
-      network_size: 64
-      attn_heads: 4
-      attn_dim_head: 64
-      dim_mults:
-      - 1
-      - 2
-      - 4
-      - 8
-      resolution: ${dataset.resolution}
-      attn_resolutions:
-      - 16
-      - 32
-      - 64
-      - 128
-      use_init_temporal_attn: true
-      use_linear_attn: true
-      time_emb_type: rotary
-  n_frames: ${dataset.n_frames}
-  metadata: ${dataset.metadata}
-  action_cond_dim: 25
-  use_plucker: true
-  _name: dememwm_memory_dit
-  memory_token_cross_attention: true
-  memory_cross_attn_layers:
-  - 3
-  - 7
-  - 11
-  - 15
-  pose_cond_dim: 5
-  log_video: true
-  dememwm:
-    enabled: true
-    training_stage: stage_1
-    debug_force_all_streams: false
-    curriculum:
-      enabled: true
-      full_stage_start_step: 20000
-      freeze_vae: true
-      dit_freeze:
-        enabled: true
-      lr:
-        dememwm_modules: 4.0e-05
-        memory_adapters: 4.0e-05
-        full_dit: 1.0e-05
-    spatial_kernel_size: 3
-    anchor:
-      enabled: true
-      anchor_indices:
-      - 0
-      - 1
-      - 2
-      - 3
-      allow_generated_as_anchor: false
-      diverse_selection: true
-      compress:
-        downsample_ratio: 2
-    dynamic:
-      enabled: true
-      recent_frames: 4
-      conv_kernel_t: 3
-      conv_stride_t: 2
-      compress:
-        downsample_ratio: 2
-    revisit:
-      enabled: true
-      deterministic_pose_retrieval: true
-      fov_overlap_threshold: 0.6
-      plucker_weight: 0.1
-      max_frames: 2
-      fov_half_h: 52.5
-      fov_half_v: 37.5
-      fov_radius: 30.0
-      fov_yaw_samples: 25
-      fov_pitch_samples: 20
-      fov_depth_samples: 20
-      pose_preselect_topk: 64
-      plucker_grid_h: 4
-      plucker_grid_w: 4
-      plucker_focal_length: 0.35
-      compress:
-        downsample_ratio: 2
-    stage_policy:
-      noise_bucket_logging: true
-    eval_ablation:
-      enabled: false
-      branch: A_plus_D_plus_R_normal
-    generated_history_proxy:
-      enabled: true
-      start_step: 40000
-      ramp_steps: 40000
-      max_prob: 0.25
-      noise_std: 0.25
-      dropout_prob: 0.0
-    injection:
-      memory_hidden_size: 512
-    cache:
-      enabled: true
-      device: cpu
-      keep_raw_latents: all
-      keep_compressed_records: true
-      keep_prefix_anchors: true
-      eviction_policy: none
-      no_evict: true
-      clear_between_videos: true
-      max_records: null
-      on_capacity_exceeded: warn
-    checkpoint:
-      strict_dememwm_eval_load: true
-  n_tokens: 4
-debug: false
-wandb:
-  entity: turlin
-  project: worldmem
-  mode: offline
-resume: null
-load: null
-auto_resume: true
-resume_ckpt_path: null
-name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-customized_load: true
-seperate_load: true
-diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-only_tune_memory: false

outputs/2026-06-03/20-22-54/.hydra/hydra.yaml

@@ -1,232 +0,0 @@
-hydra:
-  run:
-    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
-  sweep:
-    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
-    subdir: ${hydra.job.num}
-  launcher:
-    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
-  sweeper:
-    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
-    max_batch_size: null
-    params: null
-  help:
-    app_name: ${hydra.job.name}
-    header: '${hydra.help.app_name} is powered by Hydra.
-
-      '
-    footer: 'Powered by Hydra (https://hydra.cc)
-
-      Use --hydra-help to view Hydra specific help
-
-      '
-    template: '${hydra.help.header}
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (group=option)
-
-
-      $APP_CONFIG_GROUPS
-
-
-      == Config ==
-
-      Override anything in the config (foo.bar=value)
-
-
-      $CONFIG
-
-
-      ${hydra.help.footer}
-
-      '
-  hydra_help:
-    template: 'Hydra (${hydra.runtime.version})
-
-      See https://hydra.cc for more info.
-
-
-      == Flags ==
-
-      $FLAGS_HELP
-
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
-      to command line)
-
-
-      $HYDRA_CONFIG_GROUPS
-
-
-      Use ''--cfg hydra'' to Show the Hydra config.
-
-      '
-    hydra_help: ???
-  hydra_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][HYDRA] %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-    root:
-      level: INFO
-      handlers:
-      - console
-    loggers:
-      logging_example:
-        level: DEBUG
-    disable_existing_loggers: false
-  job_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-      file:
-        class: logging.FileHandler
-        formatter: simple
-        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
-    root:
-      level: INFO
-      handlers:
-      - console
-      - file
-    disable_existing_loggers: false
-  env: {}
-  mode: RUN
-  searchpath: []
-  callbacks: {}
-  output_subdir: .hydra
-  overrides:
-    hydra:
-    - hydra.mode=RUN
-    task:
-    - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-    - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-    - auto_resume=true
-    - experiment.tasks=[training]
-    - algorithm=dememwm_memory_dit
-    - +customized_load=true
-    - +seperate_load=true
-    - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-    - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-    - +only_tune_memory=false
-    - dataset=video_minecraft_latent
-    - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-    - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-    - dataset.n_frames=1000
-    - +dataset.n_frames_valid=1100
-    - +dataset.customized_validation=true
-    - +dataset.wo_updown=false
-    - +dataset.angle_range=180
-    - +dataset.pos_range=8
-    - ++algorithm.n_tokens=4
-    - algorithm.x_shape=[16,18,32]
-    - ++algorithm.context_frames=100
-    - ++algorithm.log_video=true
-    - ++algorithm.diffusion.sampling_timesteps=20
-    - ++algorithm.dememwm.debug_force_all_streams=false
-    - ++algorithm.dememwm.generated_history_proxy.enabled=true
-    - ++algorithm.dememwm.generated_history_proxy.start_step=40000
-    - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-    - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-    - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-    - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-    - ++algorithm.dememwm.anchor.enabled=true
-    - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-    - ++algorithm.dememwm.anchor.diverse_selection=true
-    - ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-    - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-    - ++algorithm.dememwm.dynamic.enabled=true
-    - ++algorithm.dememwm.dynamic.recent_frames=4
-    - ++algorithm.dememwm.dynamic.conv_stride_t=2
-    - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-    - ++algorithm.dememwm.revisit.enabled=true
-    - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-    - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-    - ++algorithm.dememwm.revisit.pose_preselect_topk=64
-    - ++algorithm.dememwm.revisit.fov_yaw_samples=25
-    - ++algorithm.dememwm.revisit.fov_pitch_samples=20
-    - ++algorithm.dememwm.revisit.fov_depth_samples=20
-    - ++algorithm.dememwm.revisit.plucker_weight=0.10
-    - ++algorithm.dememwm.revisit.max_frames=2
-    - ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-    - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-    - ++algorithm.dememwm.cache.enabled=true
-    - ++algorithm.dememwm.cache.device=cpu
-    - ++algorithm.dememwm.cache.keep_raw_latents=all
-    - ++algorithm.dememwm.cache.keep_compressed_records=true
-    - ++algorithm.dememwm.cache.eviction_policy=none
-    - ++algorithm.dememwm.cache.no_evict=true
-    - ++algorithm.dememwm.cache.clear_between_videos=true
-    - ++algorithm.dememwm.cache.max_records=null
-    - ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-    - ++algorithm.dememwm.curriculum.enabled=true
-    - ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-    - ++algorithm.dememwm.curriculum.freeze_vae=true
-    - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-    - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-    - experiment.training.batch_size=16
-    - experiment.training.optim.accumulate_grad_batches=1
-    - experiment.validation.batch_size=1
-    - experiment.validation.limit_batch=2
-    - experiment.training.checkpointing.every_n_train_steps=2000
-    - experiment.validation.val_every_n_step=2000
-    - experiment.training.max_steps=350000
-  job:
-    name: main
-    chdir: null
-    override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000
-    id: ???
-    num: ???
-    config_name: training
-    env_set: {}
-    env_copy: []
-    config:
-      override_dirname:
-        kv_sep: '='
-        item_sep: ','
-        exclude_keys: []
-  runtime:
-    version: 1.3.2
-    version_base: '1.3'
-    cwd: /proj/cvl/users/x_fahkh2/DeMemWM
-    config_sources:
-    - path: hydra.conf
-      schema: pkg
-      provider: hydra
-    - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations
-      schema: file
-      provider: main
-    - path: ''
-      schema: structured
-      provider: schema
-    output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-03/20-22-54
-    choices:
-      cluster: null
-      algorithm: dememwm_memory_dit
-      dataset: video_minecraft_latent
-      experiment: exp_video
-      hydra/env: default
-      hydra/callbacks: null
-      hydra/job_logging: default
-      hydra/hydra_logging: default
-      hydra/hydra_help: default
-      hydra/help: default
-      hydra/sweeper: basic
-      hydra/launcher: basic
-      hydra/output: default
-  verbose: false

outputs/2026-06-03/20-22-54/.hydra/overrides.yaml

@@ -1,74 +0,0 @@
-- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-- auto_resume=true
-- experiment.tasks=[training]
-- algorithm=dememwm_memory_dit
-- +customized_load=true
-- +seperate_load=true
-- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-- +only_tune_memory=false
-- dataset=video_minecraft_latent
-- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-- dataset.n_frames=1000
-- +dataset.n_frames_valid=1100
-- +dataset.customized_validation=true
-- +dataset.wo_updown=false
-- +dataset.angle_range=180
-- +dataset.pos_range=8
-- ++algorithm.n_tokens=4
-- algorithm.x_shape=[16,18,32]
-- ++algorithm.context_frames=100
-- ++algorithm.log_video=true
-- ++algorithm.diffusion.sampling_timesteps=20
-- ++algorithm.dememwm.debug_force_all_streams=false
-- ++algorithm.dememwm.generated_history_proxy.enabled=true
-- ++algorithm.dememwm.generated_history_proxy.start_step=40000
-- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-- ++algorithm.dememwm.anchor.enabled=true
-- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-- ++algorithm.dememwm.anchor.diverse_selection=true
-- ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-- ++algorithm.dememwm.dynamic.enabled=true
-- ++algorithm.dememwm.dynamic.recent_frames=4
-- ++algorithm.dememwm.dynamic.conv_stride_t=2
-- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-- ++algorithm.dememwm.revisit.enabled=true
-- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-- ++algorithm.dememwm.revisit.pose_preselect_topk=64
-- ++algorithm.dememwm.revisit.fov_yaw_samples=25
-- ++algorithm.dememwm.revisit.fov_pitch_samples=20
-- ++algorithm.dememwm.revisit.fov_depth_samples=20
-- ++algorithm.dememwm.revisit.plucker_weight=0.10
-- ++algorithm.dememwm.revisit.max_frames=2
-- ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-- ++algorithm.dememwm.cache.enabled=true
-- ++algorithm.dememwm.cache.device=cpu
-- ++algorithm.dememwm.cache.keep_raw_latents=all
-- ++algorithm.dememwm.cache.keep_compressed_records=true
-- ++algorithm.dememwm.cache.eviction_policy=none
-- ++algorithm.dememwm.cache.no_evict=true
-- ++algorithm.dememwm.cache.clear_between_videos=true
-- ++algorithm.dememwm.cache.max_records=null
-- ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-- ++algorithm.dememwm.curriculum.enabled=true
-- ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-- ++algorithm.dememwm.curriculum.freeze_vae=true
-- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-- experiment.training.batch_size=16
-- experiment.training.optim.accumulate_grad_batches=1
-- experiment.validation.batch_size=1
-- experiment.validation.limit_batch=2
-- experiment.training.checkpointing.every_n_train_steps=2000
-- experiment.validation.val_every_n_step=2000
-- experiment.training.max_steps=350000

outputs/2026-06-03/20-22-54/main.log

@@ -1,47 +0,0 @@
-[2026-06-03 20:23:36,031][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP)
-[2026-06-03 20:23:36,800][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True
-[2026-06-03 20:23:36,800][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores
-[2026-06-03 20:23:36,800][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs
-[2026-06-03 20:23:36,800][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs
-[2026-06-03 20:23:38,339][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix=''
-[2026-06-03 20:23:39,236][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix=''
-[2026-06-03 20:23:42,001][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8
-[2026-06-03 20:23:42,005][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8
-[2026-06-03 20:23:42,015][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8
-[2026-06-03 20:23:42,026][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8
-[2026-06-03 20:23:48,581][pytorch_lightning.utilities.rank_zero][INFO] - ----------------------------------------------------------------------------------------------------
-distributed_backend=nccl
-All distributed processes registered. Starting with 8 processes
-----------------------------------------------------------------------------------------------------
-
-[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-03 20:24:05,745][lightning.pytorch.callbacks.model_summary][INFO] - 
-  | Name                       | Type                                  | Params
--------------------------------------------------------------------------------------
-0 | diffusion_model            | Diffusion                             | 631 M 
-1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor         | 887 K 
-2 | dememwm_anchor_proj        | SpatialConv2DMemoryProjector          | 296 K 
-3 | dememwm_revisit_proj       | SpatialConv2DMemoryProjector          | 296 K 
-4 | dememwm_revisit_gate       | RevisitRawGate                        | 4     
-5 | validation_lpips_model     | LearnedPerceptualImagePatchSimilarity | 2.5 M 
-6 | vae                        | AutoencoderKL                         | 229 M 
--------------------------------------------------------------------------------------
-632 M     Trainable params
-231 M     Non-trainable params
-864 M     Total params
-3,456.995 Total estimated model params size (MB)
-[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15
-[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15
-[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15
-[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15
-[2026-06-04 06:21:13,814][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-04 06:21:13,814][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15

outputs/2026-06-03/20-22-55/.hydra/config.yaml

@@ -1,234 +0,0 @@
-experiment:
-  debug: ${debug}
-  tasks:
-  - training
-  num_nodes: 1
-  training:
-    precision: 16-mixed
-    compile: false
-    lr: 2.0e-05
-    batch_size: 16
-    max_epochs: -1
-    max_steps: 350000
-    max_time: null
-    data:
-      num_workers: 4
-      shuffle: true
-    optim:
-      accumulate_grad_batches: 1
-      gradient_clip_val: 1.0
-    checkpointing:
-      every_n_train_steps: 2000
-      every_n_epochs: null
-      train_time_interval: null
-      save_last: true
-      enable_version_counter: false
-  validation:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    val_every_n_step: 2000
-    val_every_n_epoch: null
-    limit_batch: 2
-    inference_mode: true
-    data:
-      num_workers: 4
-      shuffle: false
-  test:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    limit_batch: 1
-    data:
-      num_workers: 4
-      shuffle: false
-  logging:
-    metrics: null
-dataset:
-  debug: ${debug}
-  metadata: data/${dataset.name}/metadata.json
-  data_mean: 0.5
-  data_std: 0.5
-  save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-  n_frames: 1000
-  context_length: 1
-  resolution: 128
-  observation_shape:
-  - 3
-  - ${dataset.resolution}
-  - ${dataset.resolution}
-  external_cond_dim: 0
-  validation_multiplier: 1
-  frame_skip: 1
-  action_cond_dim: 25
-  _name: video_minecraft_latent
-  precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-  n_frames_valid: 1100
-  customized_validation: true
-  wo_updown: false
-  angle_range: 180
-  pos_range: 8
-algorithm:
-  debug: ${debug}
-  lr: ${experiment.training.lr}
-  x_shape:
-  - 16
-  - 18
-  - 32
-  frame_stack: 1
-  frame_skip: ${dataset.frame_skip}
-  data_mean: ${dataset.data_mean}
-  data_std: ${dataset.data_std}
-  external_cond_dim: 0
-  context_frames: 100
-  weight_decay: 0.002
-  warmup_steps: 1000
-  optimizer_beta:
-  - 0.9
-  - 0.99
-  uncertainty_scale: 1
-  guidance_scale: 0.0
-  chunk_size: 1
-  scheduling_matrix: autoregressive
-  noise_level: random_all
-  causal: true
-  diffusion:
-    objective: pred_v
-    beta_schedule: sigmoid
-    schedule_fn_kwargs: {}
-    clip_noise: 20.0
-    use_snr: false
-    use_cum_snr: false
-    use_fused_snr: true
-    snr_clip: 5.0
-    cum_snr_decay: 0.96
-    timesteps: 1000
-    sampling_timesteps: 20
-    ddim_sampling_eta: 0.0
-    stabilization_level: 15
-    architecture:
-      network_size: 64
-      attn_heads: 4
-      attn_dim_head: 64
-      dim_mults:
-      - 1
-      - 2
-      - 4
-      - 8
-      resolution: ${dataset.resolution}
-      attn_resolutions:
-      - 16
-      - 32
-      - 64
-      - 128
-      use_init_temporal_attn: true
-      use_linear_attn: true
-      time_emb_type: rotary
-  n_frames: ${dataset.n_frames}
-  metadata: ${dataset.metadata}
-  action_cond_dim: 25
-  use_plucker: true
-  _name: dememwm_memory_dit
-  memory_token_cross_attention: true
-  memory_cross_attn_layers:
-  - 3
-  - 7
-  - 11
-  - 15
-  pose_cond_dim: 5
-  log_video: true
-  dememwm:
-    enabled: true
-    training_stage: stage_1
-    debug_force_all_streams: false
-    curriculum:
-      enabled: true
-      full_stage_start_step: 20000
-      freeze_vae: true
-      dit_freeze:
-        enabled: true
-      lr:
-        dememwm_modules: 4.0e-05
-        memory_adapters: 4.0e-05
-        full_dit: 1.0e-05
-    spatial_kernel_size: 3
-    anchor:
-      enabled: true
-      anchor_indices:
-      - 0
-      - 1
-      - 2
-      - 3
-      allow_generated_as_anchor: false
-      diverse_selection: true
-      compress:
-        downsample_ratio: 2
-    dynamic:
-      enabled: true
-      recent_frames: 4
-      conv_kernel_t: 3
-      conv_stride_t: 2
-      compress:
-        downsample_ratio: 2
-    revisit:
-      enabled: true
-      deterministic_pose_retrieval: true
-      fov_overlap_threshold: 0.6
-      plucker_weight: 0.1
-      max_frames: 2
-      fov_half_h: 52.5
-      fov_half_v: 37.5
-      fov_radius: 30.0
-      fov_yaw_samples: 25
-      fov_pitch_samples: 20
-      fov_depth_samples: 20
-      pose_preselect_topk: 64
-      plucker_grid_h: 4
-      plucker_grid_w: 4
-      plucker_focal_length: 0.35
-      compress:
-        downsample_ratio: 2
-    stage_policy:
-      noise_bucket_logging: true
-    eval_ablation:
-      enabled: false
-      branch: A_plus_D_plus_R_normal
-    generated_history_proxy:
-      enabled: true
-      start_step: 40000
-      ramp_steps: 40000
-      max_prob: 0.25
-      noise_std: 0.25
-      dropout_prob: 0.0
-    injection:
-      memory_hidden_size: 512
-    cache:
-      enabled: true
-      device: cpu
-      keep_raw_latents: all
-      keep_compressed_records: true
-      keep_prefix_anchors: true
-      eviction_policy: none
-      no_evict: true
-      clear_between_videos: true
-      max_records: null
-      on_capacity_exceeded: warn
-    checkpoint:
-      strict_dememwm_eval_load: true
-  n_tokens: 4
-debug: false
-wandb:
-  entity: turlin
-  project: worldmem
-  mode: offline
-resume: null
-load: null
-auto_resume: true
-resume_ckpt_path: null
-name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-customized_load: true
-seperate_load: true
-diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-only_tune_memory: false

outputs/2026-06-03/20-22-55/.hydra/hydra.yaml

@@ -1,232 +0,0 @@
-hydra:
-  run:
-    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
-  sweep:
-    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
-    subdir: ${hydra.job.num}
-  launcher:
-    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
-  sweeper:
-    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
-    max_batch_size: null
-    params: null
-  help:
-    app_name: ${hydra.job.name}
-    header: '${hydra.help.app_name} is powered by Hydra.
-
-      '
-    footer: 'Powered by Hydra (https://hydra.cc)
-
-      Use --hydra-help to view Hydra specific help
-
-      '
-    template: '${hydra.help.header}
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (group=option)
-
-
-      $APP_CONFIG_GROUPS
-
-
-      == Config ==
-
-      Override anything in the config (foo.bar=value)
-
-
-      $CONFIG
-
-
-      ${hydra.help.footer}
-
-      '
-  hydra_help:
-    template: 'Hydra (${hydra.runtime.version})
-
-      See https://hydra.cc for more info.
-
-
-      == Flags ==
-
-      $FLAGS_HELP
-
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
-      to command line)
-
-
-      $HYDRA_CONFIG_GROUPS
-
-
-      Use ''--cfg hydra'' to Show the Hydra config.
-
-      '
-    hydra_help: ???
-  hydra_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][HYDRA] %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-    root:
-      level: INFO
-      handlers:
-      - console
-    loggers:
-      logging_example:
-        level: DEBUG
-    disable_existing_loggers: false
-  job_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-      file:
-        class: logging.FileHandler
-        formatter: simple
-        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
-    root:
-      level: INFO
-      handlers:
-      - console
-      - file
-    disable_existing_loggers: false
-  env: {}
-  mode: RUN
-  searchpath: []
-  callbacks: {}
-  output_subdir: .hydra
-  overrides:
-    hydra:
-    - hydra.mode=RUN
-    task:
-    - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-    - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-    - auto_resume=true
-    - experiment.tasks=[training]
-    - algorithm=dememwm_memory_dit
-    - +customized_load=true
-    - +seperate_load=true
-    - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-    - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-    - +only_tune_memory=false
-    - dataset=video_minecraft_latent
-    - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-    - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-    - dataset.n_frames=1000
-    - +dataset.n_frames_valid=1100
-    - +dataset.customized_validation=true
-    - +dataset.wo_updown=false
-    - +dataset.angle_range=180
-    - +dataset.pos_range=8
-    - ++algorithm.n_tokens=4
-    - algorithm.x_shape=[16,18,32]
-    - ++algorithm.context_frames=100
-    - ++algorithm.log_video=true
-    - ++algorithm.diffusion.sampling_timesteps=20
-    - ++algorithm.dememwm.debug_force_all_streams=false
-    - ++algorithm.dememwm.generated_history_proxy.enabled=true
-    - ++algorithm.dememwm.generated_history_proxy.start_step=40000
-    - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-    - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-    - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-    - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-    - ++algorithm.dememwm.anchor.enabled=true
-    - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-    - ++algorithm.dememwm.anchor.diverse_selection=true
-    - ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-    - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-    - ++algorithm.dememwm.dynamic.enabled=true
-    - ++algorithm.dememwm.dynamic.recent_frames=4
-    - ++algorithm.dememwm.dynamic.conv_stride_t=2
-    - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-    - ++algorithm.dememwm.revisit.enabled=true
-    - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-    - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-    - ++algorithm.dememwm.revisit.pose_preselect_topk=64
-    - ++algorithm.dememwm.revisit.fov_yaw_samples=25
-    - ++algorithm.dememwm.revisit.fov_pitch_samples=20
-    - ++algorithm.dememwm.revisit.fov_depth_samples=20
-    - ++algorithm.dememwm.revisit.plucker_weight=0.10
-    - ++algorithm.dememwm.revisit.max_frames=2
-    - ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-    - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-    - ++algorithm.dememwm.cache.enabled=true
-    - ++algorithm.dememwm.cache.device=cpu
-    - ++algorithm.dememwm.cache.keep_raw_latents=all
-    - ++algorithm.dememwm.cache.keep_compressed_records=true
-    - ++algorithm.dememwm.cache.eviction_policy=none
-    - ++algorithm.dememwm.cache.no_evict=true
-    - ++algorithm.dememwm.cache.clear_between_videos=true
-    - ++algorithm.dememwm.cache.max_records=null
-    - ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-    - ++algorithm.dememwm.curriculum.enabled=true
-    - ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-    - ++algorithm.dememwm.curriculum.freeze_vae=true
-    - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-    - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-    - experiment.training.batch_size=16
-    - experiment.training.optim.accumulate_grad_batches=1
-    - experiment.validation.batch_size=1
-    - experiment.validation.limit_batch=2
-    - experiment.training.checkpointing.every_n_train_steps=2000
-    - experiment.validation.val_every_n_step=2000
-    - experiment.training.max_steps=350000
-  job:
-    name: main
-    chdir: null
-    override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000
-    id: ???
-    num: ???
-    config_name: training
-    env_set: {}
-    env_copy: []
-    config:
-      override_dirname:
-        kv_sep: '='
-        item_sep: ','
-        exclude_keys: []
-  runtime:
-    version: 1.3.2
-    version_base: '1.3'
-    cwd: /proj/cvl/users/x_fahkh2/DeMemWM
-    config_sources:
-    - path: hydra.conf
-      schema: pkg
-      provider: hydra
-    - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations
-      schema: file
-      provider: main
-    - path: ''
-      schema: structured
-      provider: schema
-    output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-03/20-22-55
-    choices:
-      cluster: null
-      algorithm: dememwm_memory_dit
-      dataset: video_minecraft_latent
-      experiment: exp_video
-      hydra/env: default
-      hydra/callbacks: null
-      hydra/job_logging: default
-      hydra/hydra_logging: default
-      hydra/hydra_help: default
-      hydra/help: default
-      hydra/sweeper: basic
-      hydra/launcher: basic
-      hydra/output: default
-  verbose: false

outputs/2026-06-03/20-22-55/.hydra/overrides.yaml

@@ -1,74 +0,0 @@
-- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-- auto_resume=true
-- experiment.tasks=[training]
-- algorithm=dememwm_memory_dit
-- +customized_load=true
-- +seperate_load=true
-- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-- +only_tune_memory=false
-- dataset=video_minecraft_latent
-- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-- dataset.n_frames=1000
-- +dataset.n_frames_valid=1100
-- +dataset.customized_validation=true
-- +dataset.wo_updown=false
-- +dataset.angle_range=180
-- +dataset.pos_range=8
-- ++algorithm.n_tokens=4
-- algorithm.x_shape=[16,18,32]
-- ++algorithm.context_frames=100
-- ++algorithm.log_video=true
-- ++algorithm.diffusion.sampling_timesteps=20
-- ++algorithm.dememwm.debug_force_all_streams=false
-- ++algorithm.dememwm.generated_history_proxy.enabled=true
-- ++algorithm.dememwm.generated_history_proxy.start_step=40000
-- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-- ++algorithm.dememwm.anchor.enabled=true
-- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-- ++algorithm.dememwm.anchor.diverse_selection=true
-- ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-- ++algorithm.dememwm.dynamic.enabled=true
-- ++algorithm.dememwm.dynamic.recent_frames=4
-- ++algorithm.dememwm.dynamic.conv_stride_t=2
-- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-- ++algorithm.dememwm.revisit.enabled=true
-- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-- ++algorithm.dememwm.revisit.pose_preselect_topk=64
-- ++algorithm.dememwm.revisit.fov_yaw_samples=25
-- ++algorithm.dememwm.revisit.fov_pitch_samples=20
-- ++algorithm.dememwm.revisit.fov_depth_samples=20
-- ++algorithm.dememwm.revisit.plucker_weight=0.10
-- ++algorithm.dememwm.revisit.max_frames=2
-- ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-- ++algorithm.dememwm.cache.enabled=true
-- ++algorithm.dememwm.cache.device=cpu
-- ++algorithm.dememwm.cache.keep_raw_latents=all
-- ++algorithm.dememwm.cache.keep_compressed_records=true
-- ++algorithm.dememwm.cache.eviction_policy=none
-- ++algorithm.dememwm.cache.no_evict=true
-- ++algorithm.dememwm.cache.clear_between_videos=true
-- ++algorithm.dememwm.cache.max_records=null
-- ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-- ++algorithm.dememwm.curriculum.enabled=true
-- ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-- ++algorithm.dememwm.curriculum.freeze_vae=true
-- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-- experiment.training.batch_size=16
-- experiment.training.optim.accumulate_grad_batches=1
-- experiment.validation.batch_size=1
-- experiment.validation.limit_batch=2
-- experiment.training.checkpointing.every_n_train_steps=2000
-- experiment.validation.val_every_n_step=2000
-- experiment.training.max_steps=350000

outputs/2026-06-03/20-22-55/main.log

@@ -1,10 +0,0 @@
-[2026-06-03 20:23:41,996][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8
-[2026-06-03 20:23:42,011][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8
-[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15
-[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15
-[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15

outputs/2026-06-03/20-22-56/.hydra/config.yaml

@@ -1,234 +0,0 @@
-experiment:
-  debug: ${debug}
-  tasks:
-  - training
-  num_nodes: 1
-  training:
-    precision: 16-mixed
-    compile: false
-    lr: 2.0e-05
-    batch_size: 16
-    max_epochs: -1
-    max_steps: 350000
-    max_time: null
-    data:
-      num_workers: 4
-      shuffle: true
-    optim:
-      accumulate_grad_batches: 1
-      gradient_clip_val: 1.0
-    checkpointing:
-      every_n_train_steps: 2000
-      every_n_epochs: null
-      train_time_interval: null
-      save_last: true
-      enable_version_counter: false
-  validation:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    val_every_n_step: 2000
-    val_every_n_epoch: null
-    limit_batch: 2
-    inference_mode: true
-    data:
-      num_workers: 4
-      shuffle: false
-  test:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    limit_batch: 1
-    data:
-      num_workers: 4
-      shuffle: false
-  logging:
-    metrics: null
-dataset:
-  debug: ${debug}
-  metadata: data/${dataset.name}/metadata.json
-  data_mean: 0.5
-  data_std: 0.5
-  save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-  n_frames: 1000
-  context_length: 1
-  resolution: 128
-  observation_shape:
-  - 3
-  - ${dataset.resolution}
-  - ${dataset.resolution}
-  external_cond_dim: 0
-  validation_multiplier: 1
-  frame_skip: 1
-  action_cond_dim: 25
-  _name: video_minecraft_latent
-  precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-  n_frames_valid: 1100
-  customized_validation: true
-  wo_updown: false
-  angle_range: 180
-  pos_range: 8
-algorithm:
-  debug: ${debug}
-  lr: ${experiment.training.lr}
-  x_shape:
-  - 16
-  - 18
-  - 32
-  frame_stack: 1
-  frame_skip: ${dataset.frame_skip}
-  data_mean: ${dataset.data_mean}
-  data_std: ${dataset.data_std}
-  external_cond_dim: 0
-  context_frames: 100
-  weight_decay: 0.002
-  warmup_steps: 1000
-  optimizer_beta:
-  - 0.9
-  - 0.99
-  uncertainty_scale: 1
-  guidance_scale: 0.0
-  chunk_size: 1
-  scheduling_matrix: autoregressive
-  noise_level: random_all
-  causal: true
-  diffusion:
-    objective: pred_v
-    beta_schedule: sigmoid
-    schedule_fn_kwargs: {}
-    clip_noise: 20.0
-    use_snr: false
-    use_cum_snr: false
-    use_fused_snr: true
-    snr_clip: 5.0
-    cum_snr_decay: 0.96
-    timesteps: 1000
-    sampling_timesteps: 20
-    ddim_sampling_eta: 0.0
-    stabilization_level: 15
-    architecture:
-      network_size: 64
-      attn_heads: 4
-      attn_dim_head: 64
-      dim_mults:
-      - 1
-      - 2
-      - 4
-      - 8
-      resolution: ${dataset.resolution}
-      attn_resolutions:
-      - 16
-      - 32
-      - 64
-      - 128
-      use_init_temporal_attn: true
-      use_linear_attn: true
-      time_emb_type: rotary
-  n_frames: ${dataset.n_frames}
-  metadata: ${dataset.metadata}
-  action_cond_dim: 25
-  use_plucker: true
-  _name: dememwm_memory_dit
-  memory_token_cross_attention: true
-  memory_cross_attn_layers:
-  - 3
-  - 7
-  - 11
-  - 15
-  pose_cond_dim: 5
-  log_video: true
-  dememwm:
-    enabled: true
-    training_stage: stage_1
-    debug_force_all_streams: false
-    curriculum:
-      enabled: true
-      full_stage_start_step: 20000
-      freeze_vae: true
-      dit_freeze:
-        enabled: true
-      lr:
-        dememwm_modules: 4.0e-05
-        memory_adapters: 4.0e-05
-        full_dit: 1.0e-05
-    spatial_kernel_size: 3
-    anchor:
-      enabled: true
-      anchor_indices:
-      - 0
-      - 1
-      - 2
-      - 3
-      allow_generated_as_anchor: false
-      diverse_selection: true
-      compress:
-        downsample_ratio: 2
-    dynamic:
-      enabled: true
-      recent_frames: 4
-      conv_kernel_t: 3
-      conv_stride_t: 2
-      compress:
-        downsample_ratio: 2
-    revisit:
-      enabled: true
-      deterministic_pose_retrieval: true
-      fov_overlap_threshold: 0.6
-      plucker_weight: 0.1
-      max_frames: 2
-      fov_half_h: 52.5
-      fov_half_v: 37.5
-      fov_radius: 30.0
-      fov_yaw_samples: 25
-      fov_pitch_samples: 20
-      fov_depth_samples: 20
-      pose_preselect_topk: 64
-      plucker_grid_h: 4
-      plucker_grid_w: 4
-      plucker_focal_length: 0.35
-      compress:
-        downsample_ratio: 2
-    stage_policy:
-      noise_bucket_logging: true
-    eval_ablation:
-      enabled: false
-      branch: A_plus_D_plus_R_normal
-    generated_history_proxy:
-      enabled: true
-      start_step: 40000
-      ramp_steps: 40000
-      max_prob: 0.25
-      noise_std: 0.25
-      dropout_prob: 0.0
-    injection:
-      memory_hidden_size: 512
-    cache:
-      enabled: true
-      device: cpu
-      keep_raw_latents: all
-      keep_compressed_records: true
-      keep_prefix_anchors: true
-      eviction_policy: none
-      no_evict: true
-      clear_between_videos: true
-      max_records: null
-      on_capacity_exceeded: warn
-    checkpoint:
-      strict_dememwm_eval_load: true
-  n_tokens: 4
-debug: false
-wandb:
-  entity: turlin
-  project: worldmem
-  mode: offline
-resume: null
-load: null
-auto_resume: true
-resume_ckpt_path: null
-name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-customized_load: true
-seperate_load: true
-diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-only_tune_memory: false

outputs/2026-06-03/20-22-56/.hydra/hydra.yaml

@@ -1,232 +0,0 @@
-hydra:
-  run:
-    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
-  sweep:
-    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
-    subdir: ${hydra.job.num}
-  launcher:
-    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
-  sweeper:
-    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
-    max_batch_size: null
-    params: null
-  help:
-    app_name: ${hydra.job.name}
-    header: '${hydra.help.app_name} is powered by Hydra.
-
-      '
-    footer: 'Powered by Hydra (https://hydra.cc)
-
-      Use --hydra-help to view Hydra specific help
-
-      '
-    template: '${hydra.help.header}
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (group=option)
-
-
-      $APP_CONFIG_GROUPS
-
-
-      == Config ==
-
-      Override anything in the config (foo.bar=value)
-
-
-      $CONFIG
-
-
-      ${hydra.help.footer}
-
-      '
-  hydra_help:
-    template: 'Hydra (${hydra.runtime.version})
-
-      See https://hydra.cc for more info.
-
-
-      == Flags ==
-
-      $FLAGS_HELP
-
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
-      to command line)
-
-
-      $HYDRA_CONFIG_GROUPS
-
-
-      Use ''--cfg hydra'' to Show the Hydra config.
-
-      '
-    hydra_help: ???
-  hydra_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][HYDRA] %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-    root:
-      level: INFO
-      handlers:
-      - console
-    loggers:
-      logging_example:
-        level: DEBUG
-    disable_existing_loggers: false
-  job_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-      file:
-        class: logging.FileHandler
-        formatter: simple
-        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
-    root:
-      level: INFO
-      handlers:
-      - console
-      - file
-    disable_existing_loggers: false
-  env: {}
-  mode: RUN
-  searchpath: []
-  callbacks: {}
-  output_subdir: .hydra
-  overrides:
-    hydra:
-    - hydra.mode=RUN
-    task:
-    - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-    - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-    - auto_resume=true
-    - experiment.tasks=[training]
-    - algorithm=dememwm_memory_dit
-    - +customized_load=true
-    - +seperate_load=true
-    - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-    - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-    - +only_tune_memory=false
-    - dataset=video_minecraft_latent
-    - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-    - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-    - dataset.n_frames=1000
-    - +dataset.n_frames_valid=1100
-    - +dataset.customized_validation=true
-    - +dataset.wo_updown=false
-    - +dataset.angle_range=180
-    - +dataset.pos_range=8
-    - ++algorithm.n_tokens=4
-    - algorithm.x_shape=[16,18,32]
-    - ++algorithm.context_frames=100
-    - ++algorithm.log_video=true
-    - ++algorithm.diffusion.sampling_timesteps=20
-    - ++algorithm.dememwm.debug_force_all_streams=false
-    - ++algorithm.dememwm.generated_history_proxy.enabled=true
-    - ++algorithm.dememwm.generated_history_proxy.start_step=40000
-    - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-    - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-    - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-    - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-    - ++algorithm.dememwm.anchor.enabled=true
-    - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-    - ++algorithm.dememwm.anchor.diverse_selection=true
-    - ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-    - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-    - ++algorithm.dememwm.dynamic.enabled=true
-    - ++algorithm.dememwm.dynamic.recent_frames=4
-    - ++algorithm.dememwm.dynamic.conv_stride_t=2
-    - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-    - ++algorithm.dememwm.revisit.enabled=true
-    - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-    - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-    - ++algorithm.dememwm.revisit.pose_preselect_topk=64
-    - ++algorithm.dememwm.revisit.fov_yaw_samples=25
-    - ++algorithm.dememwm.revisit.fov_pitch_samples=20
-    - ++algorithm.dememwm.revisit.fov_depth_samples=20
-    - ++algorithm.dememwm.revisit.plucker_weight=0.10
-    - ++algorithm.dememwm.revisit.max_frames=2
-    - ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-    - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-    - ++algorithm.dememwm.cache.enabled=true
-    - ++algorithm.dememwm.cache.device=cpu
-    - ++algorithm.dememwm.cache.keep_raw_latents=all
-    - ++algorithm.dememwm.cache.keep_compressed_records=true
-    - ++algorithm.dememwm.cache.eviction_policy=none
-    - ++algorithm.dememwm.cache.no_evict=true
-    - ++algorithm.dememwm.cache.clear_between_videos=true
-    - ++algorithm.dememwm.cache.max_records=null
-    - ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-    - ++algorithm.dememwm.curriculum.enabled=true
-    - ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-    - ++algorithm.dememwm.curriculum.freeze_vae=true
-    - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-    - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-    - experiment.training.batch_size=16
-    - experiment.training.optim.accumulate_grad_batches=1
-    - experiment.validation.batch_size=1
-    - experiment.validation.limit_batch=2
-    - experiment.training.checkpointing.every_n_train_steps=2000
-    - experiment.validation.val_every_n_step=2000
-    - experiment.training.max_steps=350000
-  job:
-    name: main
-    chdir: null
-    override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000
-    id: ???
-    num: ???
-    config_name: training
-    env_set: {}
-    env_copy: []
-    config:
-      override_dirname:
-        kv_sep: '='
-        item_sep: ','
-        exclude_keys: []
-  runtime:
-    version: 1.3.2
-    version_base: '1.3'
-    cwd: /proj/cvl/users/x_fahkh2/DeMemWM
-    config_sources:
-    - path: hydra.conf
-      schema: pkg
-      provider: hydra
-    - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations
-      schema: file
-      provider: main
-    - path: ''
-      schema: structured
-      provider: schema
-    output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-03/20-22-56
-    choices:
-      cluster: null
-      algorithm: dememwm_memory_dit
-      dataset: video_minecraft_latent
-      experiment: exp_video
-      hydra/env: default
-      hydra/callbacks: null
-      hydra/job_logging: default
-      hydra/hydra_logging: default
-      hydra/hydra_help: default
-      hydra/help: default
-      hydra/sweeper: basic
-      hydra/launcher: basic
-      hydra/output: default
-  verbose: false

outputs/2026-06-03/20-22-56/.hydra/overrides.yaml

@@ -1,74 +0,0 @@
-- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-- auto_resume=true
-- experiment.tasks=[training]
-- algorithm=dememwm_memory_dit
-- +customized_load=true
-- +seperate_load=true
-- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-- +only_tune_memory=false
-- dataset=video_minecraft_latent
-- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-- dataset.n_frames=1000
-- +dataset.n_frames_valid=1100
-- +dataset.customized_validation=true
-- +dataset.wo_updown=false
-- +dataset.angle_range=180
-- +dataset.pos_range=8
-- ++algorithm.n_tokens=4
-- algorithm.x_shape=[16,18,32]
-- ++algorithm.context_frames=100
-- ++algorithm.log_video=true
-- ++algorithm.diffusion.sampling_timesteps=20
-- ++algorithm.dememwm.debug_force_all_streams=false
-- ++algorithm.dememwm.generated_history_proxy.enabled=true
-- ++algorithm.dememwm.generated_history_proxy.start_step=40000
-- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-- ++algorithm.dememwm.anchor.enabled=true
-- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-- ++algorithm.dememwm.anchor.diverse_selection=true
-- ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-- ++algorithm.dememwm.dynamic.enabled=true
-- ++algorithm.dememwm.dynamic.recent_frames=4
-- ++algorithm.dememwm.dynamic.conv_stride_t=2
-- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-- ++algorithm.dememwm.revisit.enabled=true
-- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-- ++algorithm.dememwm.revisit.pose_preselect_topk=64
-- ++algorithm.dememwm.revisit.fov_yaw_samples=25
-- ++algorithm.dememwm.revisit.fov_pitch_samples=20
-- ++algorithm.dememwm.revisit.fov_depth_samples=20
-- ++algorithm.dememwm.revisit.plucker_weight=0.10
-- ++algorithm.dememwm.revisit.max_frames=2
-- ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true
-- ++algorithm.dememwm.cache.enabled=true
-- ++algorithm.dememwm.cache.device=cpu
-- ++algorithm.dememwm.cache.keep_raw_latents=all
-- ++algorithm.dememwm.cache.keep_compressed_records=true
-- ++algorithm.dememwm.cache.eviction_policy=none
-- ++algorithm.dememwm.cache.no_evict=true
-- ++algorithm.dememwm.cache.clear_between_videos=true
-- ++algorithm.dememwm.cache.max_records=null
-- ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-- ++algorithm.dememwm.curriculum.enabled=true
-- ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-- ++algorithm.dememwm.curriculum.freeze_vae=true
-- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-- experiment.training.batch_size=16
-- experiment.training.optim.accumulate_grad_batches=1
-- experiment.validation.batch_size=1
-- experiment.validation.limit_batch=2
-- experiment.training.checkpointing.every_n_train_steps=2000
-- experiment.validation.val_every_n_step=2000
-- experiment.training.max_steps=350000

outputs/2026-06-03/20-22-56/main.log

@@ -1,10 +0,0 @@
-[2026-06-03 20:23:42,010][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8
-[2026-06-03 20:23:42,017][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8
-[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15
-[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15
-[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15

outputs/2026-06-04/06-22-27/.hydra/config.yaml

@@ -1,231 +0,0 @@
-experiment:
-  debug: ${debug}
-  tasks:
-  - training
-  num_nodes: 1
-  training:
-    precision: 16-mixed
-    compile: false
-    lr: 2.0e-05
-    batch_size: 16
-    max_epochs: -1
-    max_steps: 350000
-    max_time: null
-    data:
-      num_workers: 4
-      shuffle: true
-    optim:
-      accumulate_grad_batches: 1
-      gradient_clip_val: 1.0
-    checkpointing:
-      every_n_train_steps: 5000
-      every_n_epochs: null
-      train_time_interval: null
-      save_last: true
-      enable_version_counter: false
-  validation:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    val_every_n_step: 5000
-    val_every_n_epoch: null
-    limit_batch: 1
-    inference_mode: true
-    data:
-      num_workers: 4
-      shuffle: false
-  test:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    limit_batch: 1
-    data:
-      num_workers: 4
-      shuffle: false
-  logging:
-    metrics: null
-dataset:
-  debug: ${debug}
-  metadata: data/${dataset.name}/metadata.json
-  data_mean: 0.5
-  data_std: 0.5
-  save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-  n_frames: 1000
-  context_length: 1
-  resolution: 128
-  observation_shape:
-  - 3
-  - ${dataset.resolution}
-  - ${dataset.resolution}
-  external_cond_dim: 0
-  validation_multiplier: 1
-  frame_skip: 1
-  action_cond_dim: 25
-  _name: video_minecraft_latent
-  precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-  n_frames_valid: 500
-  customized_validation: true
-  wo_updown: false
-  angle_range: 180
-  pos_range: 8
-algorithm:
-  debug: ${debug}
-  lr: ${experiment.training.lr}
-  x_shape:
-  - 16
-  - 18
-  - 32
-  frame_stack: 1
-  frame_skip: ${dataset.frame_skip}
-  data_mean: ${dataset.data_mean}
-  data_std: ${dataset.data_std}
-  external_cond_dim: 0
-  context_frames: 100
-  weight_decay: 0.002
-  warmup_steps: 1000
-  optimizer_beta:
-  - 0.9
-  - 0.99
-  uncertainty_scale: 1
-  guidance_scale: 0.0
-  chunk_size: 1
-  scheduling_matrix: autoregressive
-  noise_level: random_all
-  causal: true
-  diffusion:
-    objective: pred_v
-    beta_schedule: sigmoid
-    schedule_fn_kwargs: {}
-    clip_noise: 20.0
-    use_snr: false
-    use_cum_snr: false
-    use_fused_snr: true
-    snr_clip: 5.0
-    cum_snr_decay: 0.96
-    timesteps: 1000
-    sampling_timesteps: 20
-    ddim_sampling_eta: 0.0
-    stabilization_level: 15
-    architecture:
-      network_size: 64
-      attn_heads: 4
-      attn_dim_head: 64
-      dim_mults:
-      - 1
-      - 2
-      - 4
-      - 8
-      resolution: ${dataset.resolution}
-      attn_resolutions:
-      - 16
-      - 32
-      - 64
-      - 128
-      use_init_temporal_attn: true
-      use_linear_attn: true
-      time_emb_type: rotary
-  n_frames: ${dataset.n_frames}
-  metadata: ${dataset.metadata}
-  action_cond_dim: 25
-  use_plucker: true
-  _name: dememwm_memory_dit
-  memory_token_cross_attention: true
-  memory_cross_attn_layers:
-  - 3
-  - 7
-  - 11
-  - 15
-  pose_cond_dim: 5
-  log_video: true
-  dememwm:
-    enabled: true
-    training_stage: stage_1
-    curriculum:
-      enabled: true
-      full_stage_start_step: 20000
-      freeze_vae: true
-      dit_freeze:
-        enabled: true
-      lr:
-        dememwm_modules: 4.0e-05
-        memory_adapters: 4.0e-05
-        full_dit: 1.0e-05
-    spatial_kernel_size: 3
-    anchor:
-      enabled: true
-      anchor_indices:
-      - 0
-      - 1
-      - 2
-      - 3
-      allow_generated_as_anchor: false
-      diverse_selection: true
-      compress:
-        downsample_ratio: 2
-    dynamic:
-      enabled: true
-      recent_frames: 4
-      conv_kernel_t: 3
-      conv_stride_t: 2
-      compress:
-        downsample_ratio: 2
-    revisit:
-      enabled: true
-      deterministic_pose_retrieval: true
-      fov_overlap_threshold: 0.6
-      plucker_weight: 0.1
-      max_frames: 2
-      fov_half_h: 52.5
-      fov_half_v: 37.5
-      fov_radius: 30.0
-      fov_yaw_samples: 25
-      fov_pitch_samples: 20
-      fov_depth_samples: 20
-      pose_preselect_topk: 64
-      plucker_grid_h: 4
-      plucker_grid_w: 4
-      plucker_focal_length: 0.35
-      compress:
-        downsample_ratio: 2
-    eval_ablation:
-      enabled: false
-      branch: A_plus_D_plus_R_normal
-    generated_history_proxy:
-      enabled: true
-      start_step: 40000
-      ramp_steps: 40000
-      max_prob: 0.25
-      noise_std: 0.25
-      dropout_prob: 0.0
-    injection:
-      memory_hidden_size: 512
-    cache:
-      enabled: true
-      device: cpu
-      keep_raw_latents: all
-      keep_compressed_records: true
-      keep_prefix_anchors: true
-      eviction_policy: none
-      no_evict: true
-      clear_between_videos: true
-      max_records: null
-      on_capacity_exceeded: warn
-    checkpoint:
-      strict_dememwm_eval_load: true
-  n_tokens: 4
-debug: false
-wandb:
-  entity: turlin
-  project: worldmem
-  mode: offline
-resume: null
-load: null
-auto_resume: true
-resume_ckpt_path: null
-name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-customized_load: true
-seperate_load: true
-diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-only_tune_memory: false

outputs/2026-06-04/06-22-27/.hydra/hydra.yaml

@@ -1,230 +0,0 @@
-hydra:
-  run:
-    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
-  sweep:
-    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
-    subdir: ${hydra.job.num}
-  launcher:
-    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
-  sweeper:
-    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
-    max_batch_size: null
-    params: null
-  help:
-    app_name: ${hydra.job.name}
-    header: '${hydra.help.app_name} is powered by Hydra.
-
-      '
-    footer: 'Powered by Hydra (https://hydra.cc)
-
-      Use --hydra-help to view Hydra specific help
-
-      '
-    template: '${hydra.help.header}
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (group=option)
-
-
-      $APP_CONFIG_GROUPS
-
-
-      == Config ==
-
-      Override anything in the config (foo.bar=value)
-
-
-      $CONFIG
-
-
-      ${hydra.help.footer}
-
-      '
-  hydra_help:
-    template: 'Hydra (${hydra.runtime.version})
-
-      See https://hydra.cc for more info.
-
-
-      == Flags ==
-
-      $FLAGS_HELP
-
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
-      to command line)
-
-
-      $HYDRA_CONFIG_GROUPS
-
-
-      Use ''--cfg hydra'' to Show the Hydra config.
-
-      '
-    hydra_help: ???
-  hydra_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][HYDRA] %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-    root:
-      level: INFO
-      handlers:
-      - console
-    loggers:
-      logging_example:
-        level: DEBUG
-    disable_existing_loggers: false
-  job_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-      file:
-        class: logging.FileHandler
-        formatter: simple
-        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
-    root:
-      level: INFO
-      handlers:
-      - console
-      - file
-    disable_existing_loggers: false
-  env: {}
-  mode: RUN
-  searchpath: []
-  callbacks: {}
-  output_subdir: .hydra
-  overrides:
-    hydra:
-    - hydra.mode=RUN
-    task:
-    - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-    - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-    - auto_resume=true
-    - experiment.tasks=[training]
-    - algorithm=dememwm_memory_dit
-    - +customized_load=true
-    - +seperate_load=true
-    - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-    - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-    - +only_tune_memory=false
-    - dataset=video_minecraft_latent
-    - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-    - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-    - dataset.n_frames=1000
-    - +dataset.n_frames_valid=500
-    - +dataset.customized_validation=true
-    - +dataset.wo_updown=false
-    - +dataset.angle_range=180
-    - +dataset.pos_range=8
-    - ++algorithm.n_tokens=4
-    - algorithm.x_shape=[16,18,32]
-    - ++algorithm.context_frames=100
-    - ++algorithm.log_video=true
-    - ++algorithm.diffusion.sampling_timesteps=20
-    - ++algorithm.dememwm.generated_history_proxy.enabled=true
-    - ++algorithm.dememwm.generated_history_proxy.start_step=40000
-    - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-    - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-    - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-    - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-    - ++algorithm.dememwm.anchor.enabled=true
-    - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-    - ++algorithm.dememwm.anchor.diverse_selection=true
-    - ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-    - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-    - ++algorithm.dememwm.dynamic.enabled=true
-    - ++algorithm.dememwm.dynamic.recent_frames=4
-    - ++algorithm.dememwm.dynamic.conv_stride_t=2
-    - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-    - ++algorithm.dememwm.revisit.enabled=true
-    - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-    - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-    - ++algorithm.dememwm.revisit.pose_preselect_topk=64
-    - ++algorithm.dememwm.revisit.fov_yaw_samples=25
-    - ++algorithm.dememwm.revisit.fov_pitch_samples=20
-    - ++algorithm.dememwm.revisit.fov_depth_samples=20
-    - ++algorithm.dememwm.revisit.plucker_weight=0.10
-    - ++algorithm.dememwm.revisit.max_frames=2
-    - ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-    - ++algorithm.dememwm.cache.enabled=true
-    - ++algorithm.dememwm.cache.device=cpu
-    - ++algorithm.dememwm.cache.keep_raw_latents=all
-    - ++algorithm.dememwm.cache.keep_compressed_records=true
-    - ++algorithm.dememwm.cache.eviction_policy=none
-    - ++algorithm.dememwm.cache.no_evict=true
-    - ++algorithm.dememwm.cache.clear_between_videos=true
-    - ++algorithm.dememwm.cache.max_records=null
-    - ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-    - ++algorithm.dememwm.curriculum.enabled=true
-    - ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-    - ++algorithm.dememwm.curriculum.freeze_vae=true
-    - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-    - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-    - experiment.training.batch_size=16
-    - experiment.training.optim.accumulate_grad_batches=1
-    - experiment.validation.batch_size=1
-    - experiment.validation.limit_batch=1
-    - experiment.training.checkpointing.every_n_train_steps=5000
-    - experiment.validation.val_every_n_step=5000
-    - experiment.training.max_steps=350000
-  job:
-    name: main
-    chdir: null
-    override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000
-    id: ???
-    num: ???
-    config_name: training
-    env_set: {}
-    env_copy: []
-    config:
-      override_dirname:
-        kv_sep: '='
-        item_sep: ','
-        exclude_keys: []
-  runtime:
-    version: 1.3.2
-    version_base: '1.3'
-    cwd: /proj/cvl/users/x_fahkh2/DeMemWM
-    config_sources:
-    - path: hydra.conf
-      schema: pkg
-      provider: hydra
-    - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations
-      schema: file
-      provider: main
-    - path: ''
-      schema: structured
-      provider: schema
-    output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-04/06-22-27
-    choices:
-      cluster: null
-      algorithm: dememwm_memory_dit
-      dataset: video_minecraft_latent
-      experiment: exp_video
-      hydra/env: default
-      hydra/callbacks: null
-      hydra/job_logging: default
-      hydra/hydra_logging: default
-      hydra/hydra_help: default
-      hydra/help: default
-      hydra/sweeper: basic
-      hydra/launcher: basic
-      hydra/output: default
-  verbose: false

outputs/2026-06-04/06-22-27/.hydra/overrides.yaml

@@ -1,72 +0,0 @@
-- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-- auto_resume=true
-- experiment.tasks=[training]
-- algorithm=dememwm_memory_dit
-- +customized_load=true
-- +seperate_load=true
-- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-- +only_tune_memory=false
-- dataset=video_minecraft_latent
-- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-- dataset.n_frames=1000
-- +dataset.n_frames_valid=500
-- +dataset.customized_validation=true
-- +dataset.wo_updown=false
-- +dataset.angle_range=180
-- +dataset.pos_range=8
-- ++algorithm.n_tokens=4
-- algorithm.x_shape=[16,18,32]
-- ++algorithm.context_frames=100
-- ++algorithm.log_video=true
-- ++algorithm.diffusion.sampling_timesteps=20
-- ++algorithm.dememwm.generated_history_proxy.enabled=true
-- ++algorithm.dememwm.generated_history_proxy.start_step=40000
-- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-- ++algorithm.dememwm.anchor.enabled=true
-- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-- ++algorithm.dememwm.anchor.diverse_selection=true
-- ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-- ++algorithm.dememwm.dynamic.enabled=true
-- ++algorithm.dememwm.dynamic.recent_frames=4
-- ++algorithm.dememwm.dynamic.conv_stride_t=2
-- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-- ++algorithm.dememwm.revisit.enabled=true
-- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-- ++algorithm.dememwm.revisit.pose_preselect_topk=64
-- ++algorithm.dememwm.revisit.fov_yaw_samples=25
-- ++algorithm.dememwm.revisit.fov_pitch_samples=20
-- ++algorithm.dememwm.revisit.fov_depth_samples=20
-- ++algorithm.dememwm.revisit.plucker_weight=0.10
-- ++algorithm.dememwm.revisit.max_frames=2
-- ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-- ++algorithm.dememwm.cache.enabled=true
-- ++algorithm.dememwm.cache.device=cpu
-- ++algorithm.dememwm.cache.keep_raw_latents=all
-- ++algorithm.dememwm.cache.keep_compressed_records=true
-- ++algorithm.dememwm.cache.eviction_policy=none
-- ++algorithm.dememwm.cache.no_evict=true
-- ++algorithm.dememwm.cache.clear_between_videos=true
-- ++algorithm.dememwm.cache.max_records=null
-- ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-- ++algorithm.dememwm.curriculum.enabled=true
-- ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-- ++algorithm.dememwm.curriculum.freeze_vae=true
-- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-- experiment.training.batch_size=16
-- experiment.training.optim.accumulate_grad_batches=1
-- experiment.validation.batch_size=1
-- experiment.validation.limit_batch=1
-- experiment.training.checkpointing.every_n_train_steps=5000
-- experiment.validation.val_every_n_step=5000
-- experiment.training.max_steps=350000

outputs/2026-06-04/06-22-28/.hydra/config.yaml

@@ -1,231 +0,0 @@
-experiment:
-  debug: ${debug}
-  tasks:
-  - training
-  num_nodes: 1
-  training:
-    precision: 16-mixed
-    compile: false
-    lr: 2.0e-05
-    batch_size: 16
-    max_epochs: -1
-    max_steps: 350000
-    max_time: null
-    data:
-      num_workers: 4
-      shuffle: true
-    optim:
-      accumulate_grad_batches: 1
-      gradient_clip_val: 1.0
-    checkpointing:
-      every_n_train_steps: 5000
-      every_n_epochs: null
-      train_time_interval: null
-      save_last: true
-      enable_version_counter: false
-  validation:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    val_every_n_step: 5000
-    val_every_n_epoch: null
-    limit_batch: 1
-    inference_mode: true
-    data:
-      num_workers: 4
-      shuffle: false
-  test:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    limit_batch: 1
-    data:
-      num_workers: 4
-      shuffle: false
-  logging:
-    metrics: null
-dataset:
-  debug: ${debug}
-  metadata: data/${dataset.name}/metadata.json
-  data_mean: 0.5
-  data_std: 0.5
-  save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-  n_frames: 1000
-  context_length: 1
-  resolution: 128
-  observation_shape:
-  - 3
-  - ${dataset.resolution}
-  - ${dataset.resolution}
-  external_cond_dim: 0
-  validation_multiplier: 1
-  frame_skip: 1
-  action_cond_dim: 25
-  _name: video_minecraft_latent
-  precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-  n_frames_valid: 500
-  customized_validation: true
-  wo_updown: false
-  angle_range: 180
-  pos_range: 8
-algorithm:
-  debug: ${debug}
-  lr: ${experiment.training.lr}
-  x_shape:
-  - 16
-  - 18
-  - 32
-  frame_stack: 1
-  frame_skip: ${dataset.frame_skip}
-  data_mean: ${dataset.data_mean}
-  data_std: ${dataset.data_std}
-  external_cond_dim: 0
-  context_frames: 100
-  weight_decay: 0.002
-  warmup_steps: 1000
-  optimizer_beta:
-  - 0.9
-  - 0.99
-  uncertainty_scale: 1
-  guidance_scale: 0.0
-  chunk_size: 1
-  scheduling_matrix: autoregressive
-  noise_level: random_all
-  causal: true
-  diffusion:
-    objective: pred_v
-    beta_schedule: sigmoid
-    schedule_fn_kwargs: {}
-    clip_noise: 20.0
-    use_snr: false
-    use_cum_snr: false
-    use_fused_snr: true
-    snr_clip: 5.0
-    cum_snr_decay: 0.96
-    timesteps: 1000
-    sampling_timesteps: 20
-    ddim_sampling_eta: 0.0
-    stabilization_level: 15
-    architecture:
-      network_size: 64
-      attn_heads: 4
-      attn_dim_head: 64
-      dim_mults:
-      - 1
-      - 2
-      - 4
-      - 8
-      resolution: ${dataset.resolution}
-      attn_resolutions:
-      - 16
-      - 32
-      - 64
-      - 128
-      use_init_temporal_attn: true
-      use_linear_attn: true
-      time_emb_type: rotary
-  n_frames: ${dataset.n_frames}
-  metadata: ${dataset.metadata}
-  action_cond_dim: 25
-  use_plucker: true
-  _name: dememwm_memory_dit
-  memory_token_cross_attention: true
-  memory_cross_attn_layers:
-  - 3
-  - 7
-  - 11
-  - 15
-  pose_cond_dim: 5
-  log_video: true
-  dememwm:
-    enabled: true
-    training_stage: stage_1
-    curriculum:
-      enabled: true
-      full_stage_start_step: 20000
-      freeze_vae: true
-      dit_freeze:
-        enabled: true
-      lr:
-        dememwm_modules: 4.0e-05
-        memory_adapters: 4.0e-05
-        full_dit: 1.0e-05
-    spatial_kernel_size: 3
-    anchor:
-      enabled: true
-      anchor_indices:
-      - 0
-      - 1
-      - 2
-      - 3
-      allow_generated_as_anchor: false
-      diverse_selection: true
-      compress:
-        downsample_ratio: 2
-    dynamic:
-      enabled: true
-      recent_frames: 4
-      conv_kernel_t: 3
-      conv_stride_t: 2
-      compress:
-        downsample_ratio: 2
-    revisit:
-      enabled: true
-      deterministic_pose_retrieval: true
-      fov_overlap_threshold: 0.6
-      plucker_weight: 0.1
-      max_frames: 2
-      fov_half_h: 52.5
-      fov_half_v: 37.5
-      fov_radius: 30.0
-      fov_yaw_samples: 25
-      fov_pitch_samples: 20
-      fov_depth_samples: 20
-      pose_preselect_topk: 64
-      plucker_grid_h: 4
-      plucker_grid_w: 4
-      plucker_focal_length: 0.35
-      compress:
-        downsample_ratio: 2
-    eval_ablation:
-      enabled: false
-      branch: A_plus_D_plus_R_normal
-    generated_history_proxy:
-      enabled: true
-      start_step: 40000
-      ramp_steps: 40000
-      max_prob: 0.25
-      noise_std: 0.25
-      dropout_prob: 0.0
-    injection:
-      memory_hidden_size: 512
-    cache:
-      enabled: true
-      device: cpu
-      keep_raw_latents: all
-      keep_compressed_records: true
-      keep_prefix_anchors: true
-      eviction_policy: none
-      no_evict: true
-      clear_between_videos: true
-      max_records: null
-      on_capacity_exceeded: warn
-    checkpoint:
-      strict_dememwm_eval_load: true
-  n_tokens: 4
-debug: false
-wandb:
-  entity: turlin
-  project: worldmem
-  mode: offline
-resume: null
-load: null
-auto_resume: true
-resume_ckpt_path: null
-name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-customized_load: true
-seperate_load: true
-diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-only_tune_memory: false

outputs/2026-06-04/06-22-28/.hydra/hydra.yaml

@@ -1,230 +0,0 @@
-hydra:
-  run:
-    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
-  sweep:
-    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
-    subdir: ${hydra.job.num}
-  launcher:
-    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
-  sweeper:
-    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
-    max_batch_size: null
-    params: null
-  help:
-    app_name: ${hydra.job.name}
-    header: '${hydra.help.app_name} is powered by Hydra.
-
-      '
-    footer: 'Powered by Hydra (https://hydra.cc)
-
-      Use --hydra-help to view Hydra specific help
-
-      '
-    template: '${hydra.help.header}
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (group=option)
-
-
-      $APP_CONFIG_GROUPS
-
-
-      == Config ==
-
-      Override anything in the config (foo.bar=value)
-
-
-      $CONFIG
-
-
-      ${hydra.help.footer}
-
-      '
-  hydra_help:
-    template: 'Hydra (${hydra.runtime.version})
-
-      See https://hydra.cc for more info.
-
-
-      == Flags ==
-
-      $FLAGS_HELP
-
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
-      to command line)
-
-
-      $HYDRA_CONFIG_GROUPS
-
-
-      Use ''--cfg hydra'' to Show the Hydra config.
-
-      '
-    hydra_help: ???
-  hydra_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][HYDRA] %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-    root:
-      level: INFO
-      handlers:
-      - console
-    loggers:
-      logging_example:
-        level: DEBUG
-    disable_existing_loggers: false
-  job_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-      file:
-        class: logging.FileHandler
-        formatter: simple
-        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
-    root:
-      level: INFO
-      handlers:
-      - console
-      - file
-    disable_existing_loggers: false
-  env: {}
-  mode: RUN
-  searchpath: []
-  callbacks: {}
-  output_subdir: .hydra
-  overrides:
-    hydra:
-    - hydra.mode=RUN
-    task:
-    - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-    - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-    - auto_resume=true
-    - experiment.tasks=[training]
-    - algorithm=dememwm_memory_dit
-    - +customized_load=true
-    - +seperate_load=true
-    - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-    - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-    - +only_tune_memory=false
-    - dataset=video_minecraft_latent
-    - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-    - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-    - dataset.n_frames=1000
-    - +dataset.n_frames_valid=500
-    - +dataset.customized_validation=true
-    - +dataset.wo_updown=false
-    - +dataset.angle_range=180
-    - +dataset.pos_range=8
-    - ++algorithm.n_tokens=4
-    - algorithm.x_shape=[16,18,32]
-    - ++algorithm.context_frames=100
-    - ++algorithm.log_video=true
-    - ++algorithm.diffusion.sampling_timesteps=20
-    - ++algorithm.dememwm.generated_history_proxy.enabled=true
-    - ++algorithm.dememwm.generated_history_proxy.start_step=40000
-    - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-    - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-    - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-    - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-    - ++algorithm.dememwm.anchor.enabled=true
-    - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-    - ++algorithm.dememwm.anchor.diverse_selection=true
-    - ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-    - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-    - ++algorithm.dememwm.dynamic.enabled=true
-    - ++algorithm.dememwm.dynamic.recent_frames=4
-    - ++algorithm.dememwm.dynamic.conv_stride_t=2
-    - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-    - ++algorithm.dememwm.revisit.enabled=true
-    - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-    - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-    - ++algorithm.dememwm.revisit.pose_preselect_topk=64
-    - ++algorithm.dememwm.revisit.fov_yaw_samples=25
-    - ++algorithm.dememwm.revisit.fov_pitch_samples=20
-    - ++algorithm.dememwm.revisit.fov_depth_samples=20
-    - ++algorithm.dememwm.revisit.plucker_weight=0.10
-    - ++algorithm.dememwm.revisit.max_frames=2
-    - ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-    - ++algorithm.dememwm.cache.enabled=true
-    - ++algorithm.dememwm.cache.device=cpu
-    - ++algorithm.dememwm.cache.keep_raw_latents=all
-    - ++algorithm.dememwm.cache.keep_compressed_records=true
-    - ++algorithm.dememwm.cache.eviction_policy=none
-    - ++algorithm.dememwm.cache.no_evict=true
-    - ++algorithm.dememwm.cache.clear_between_videos=true
-    - ++algorithm.dememwm.cache.max_records=null
-    - ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-    - ++algorithm.dememwm.curriculum.enabled=true
-    - ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-    - ++algorithm.dememwm.curriculum.freeze_vae=true
-    - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-    - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-    - experiment.training.batch_size=16
-    - experiment.training.optim.accumulate_grad_batches=1
-    - experiment.validation.batch_size=1
-    - experiment.validation.limit_batch=1
-    - experiment.training.checkpointing.every_n_train_steps=5000
-    - experiment.validation.val_every_n_step=5000
-    - experiment.training.max_steps=350000
-  job:
-    name: main
-    chdir: null
-    override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000
-    id: ???
-    num: ???
-    config_name: training
-    env_set: {}
-    env_copy: []
-    config:
-      override_dirname:
-        kv_sep: '='
-        item_sep: ','
-        exclude_keys: []
-  runtime:
-    version: 1.3.2
-    version_base: '1.3'
-    cwd: /proj/cvl/users/x_fahkh2/DeMemWM
-    config_sources:
-    - path: hydra.conf
-      schema: pkg
-      provider: hydra
-    - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations
-      schema: file
-      provider: main
-    - path: ''
-      schema: structured
-      provider: schema
-    output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-04/06-22-28
-    choices:
-      cluster: null
-      algorithm: dememwm_memory_dit
-      dataset: video_minecraft_latent
-      experiment: exp_video
-      hydra/env: default
-      hydra/callbacks: null
-      hydra/job_logging: default
-      hydra/hydra_logging: default
-      hydra/hydra_help: default
-      hydra/help: default
-      hydra/sweeper: basic
-      hydra/launcher: basic
-      hydra/output: default
-  verbose: false

outputs/2026-06-04/06-22-28/.hydra/overrides.yaml

@@ -1,72 +0,0 @@
-- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k
-- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/
-- auto_resume=true
-- experiment.tasks=[training]
-- algorithm=dememwm_memory_dit
-- +customized_load=true
-- +seperate_load=true
-- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-- +only_tune_memory=false
-- dataset=video_minecraft_latent
-- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-- dataset.n_frames=1000
-- +dataset.n_frames_valid=500
-- +dataset.customized_validation=true
-- +dataset.wo_updown=false
-- +dataset.angle_range=180
-- +dataset.pos_range=8
-- ++algorithm.n_tokens=4
-- algorithm.x_shape=[16,18,32]
-- ++algorithm.context_frames=100
-- ++algorithm.log_video=true
-- ++algorithm.diffusion.sampling_timesteps=20
-- ++algorithm.dememwm.generated_history_proxy.enabled=true
-- ++algorithm.dememwm.generated_history_proxy.start_step=40000
-- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-- ++algorithm.dememwm.anchor.enabled=true
-- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-- ++algorithm.dememwm.anchor.diverse_selection=true
-- ++algorithm.dememwm.anchor.compress.downsample_ratio=2
-- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-- ++algorithm.dememwm.dynamic.enabled=true
-- ++algorithm.dememwm.dynamic.recent_frames=4
-- ++algorithm.dememwm.dynamic.conv_stride_t=2
-- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2
-- ++algorithm.dememwm.revisit.enabled=true
-- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-- ++algorithm.dememwm.revisit.pose_preselect_topk=64
-- ++algorithm.dememwm.revisit.fov_yaw_samples=25
-- ++algorithm.dememwm.revisit.fov_pitch_samples=20
-- ++algorithm.dememwm.revisit.fov_depth_samples=20
-- ++algorithm.dememwm.revisit.plucker_weight=0.10
-- ++algorithm.dememwm.revisit.max_frames=2
-- ++algorithm.dememwm.revisit.compress.downsample_ratio=2
-- ++algorithm.dememwm.cache.enabled=true
-- ++algorithm.dememwm.cache.device=cpu
-- ++algorithm.dememwm.cache.keep_raw_latents=all
-- ++algorithm.dememwm.cache.keep_compressed_records=true
-- ++algorithm.dememwm.cache.eviction_policy=none
-- ++algorithm.dememwm.cache.no_evict=true
-- ++algorithm.dememwm.cache.clear_between_videos=true
-- ++algorithm.dememwm.cache.max_records=null
-- ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-- ++algorithm.dememwm.curriculum.enabled=true
-- ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-- ++algorithm.dememwm.curriculum.freeze_vae=true
-- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-- experiment.training.batch_size=16
-- experiment.training.optim.accumulate_grad_batches=1
-- experiment.validation.batch_size=1
-- experiment.validation.limit_batch=1
-- experiment.training.checkpointing.every_n_train_steps=5000
-- experiment.validation.val_every_n_step=5000
-- experiment.training.max_steps=350000

outputs/2026-06-04/06-24-27/.hydra/config.yaml

@@ -1,232 +0,0 @@
-experiment:
-  debug: ${debug}
-  tasks:
-  - training
-  num_nodes: 1
-  training:
-    precision: 16-mixed
-    compile: false
-    lr: 2.0e-05
-    batch_size: 16
-    max_epochs: -1
-    max_steps: 350000
-    max_time: null
-    data:
-      num_workers: 4
-      shuffle: true
-    optim:
-      accumulate_grad_batches: 1
-      gradient_clip_val: 1.0
-    checkpointing:
-      every_n_train_steps: 5000
-      every_n_epochs: null
-      train_time_interval: null
-      save_last: true
-      enable_version_counter: false
-  validation:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    val_every_n_step: 5000
-    val_every_n_epoch: null
-    limit_batch: 1
-    inference_mode: true
-    data:
-      num_workers: 4
-      shuffle: false
-  test:
-    precision: 16-mixed
-    compile: false
-    batch_size: 1
-    limit_batch: 1
-    data:
-      num_workers: 4
-      shuffle: false
-  logging:
-    metrics: null
-dataset:
-  debug: ${debug}
-  metadata: data/${dataset.name}/metadata.json
-  data_mean: 0.5
-  data_std: 0.5
-  save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-  n_frames: 1000
-  context_length: 1
-  resolution: 128
-  observation_shape:
-  - 3
-  - ${dataset.resolution}
-  - ${dataset.resolution}
-  external_cond_dim: 0
-  validation_multiplier: 1
-  frame_skip: 1
-  action_cond_dim: 25
-  _name: video_minecraft_latent
-  precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-  n_frames_valid: 500
-  customized_validation: true
-  memory_condition_length: 0
-  wo_updown: false
-  angle_range: 180
-  pos_range: 8
-algorithm:
-  debug: ${debug}
-  lr: ${experiment.training.lr}
-  x_shape:
-  - 16
-  - 18
-  - 32
-  frame_stack: 1
-  frame_skip: ${dataset.frame_skip}
-  data_mean: ${dataset.data_mean}
-  data_std: ${dataset.data_std}
-  external_cond_dim: 0
-  context_frames: 100
-  weight_decay: 0.002
-  warmup_steps: 1000
-  optimizer_beta:
-  - 0.9
-  - 0.99
-  uncertainty_scale: 1
-  guidance_scale: 0.0
-  chunk_size: 1
-  scheduling_matrix: autoregressive
-  noise_level: random_all
-  causal: true
-  diffusion:
-    objective: pred_v
-    beta_schedule: sigmoid
-    schedule_fn_kwargs: {}
-    clip_noise: 20.0
-    use_snr: false
-    use_cum_snr: false
-    use_fused_snr: true
-    snr_clip: 5.0
-    cum_snr_decay: 0.96
-    timesteps: 1000
-    sampling_timesteps: 20
-    ddim_sampling_eta: 0.0
-    stabilization_level: 15
-    architecture:
-      network_size: 64
-      attn_heads: 4
-      attn_dim_head: 64
-      dim_mults:
-      - 1
-      - 2
-      - 4
-      - 8
-      resolution: ${dataset.resolution}
-      attn_resolutions:
-      - 16
-      - 32
-      - 64
-      - 128
-      use_init_temporal_attn: true
-      use_linear_attn: true
-      time_emb_type: rotary
-  n_frames: ${dataset.n_frames}
-  metadata: ${dataset.metadata}
-  action_cond_dim: 25
-  use_plucker: true
-  _name: dememwm_memory_dit
-  memory_token_cross_attention: true
-  memory_cross_attn_layers: null
-  memory_condition_length: 0
-  pose_cond_dim: 5
-  log_video: true
-  dememwm:
-    enabled: true
-    training_stage: stage_1
-    curriculum:
-      enabled: true
-      full_stage_start_step: 20000
-      freeze_vae: true
-      dit_freeze:
-        enabled: true
-      lr:
-        dememwm_modules: 4.0e-05
-        memory_adapters: 4.0e-05
-        full_dit: 1.0e-05
-    token_patch_size: 2
-    anchor:
-      enabled: true
-      anchor_indices:
-      - 0
-      - 1
-      - 2
-      - 3
-      allow_generated_as_anchor: false
-      diverse_selection: true
-      compress:
-        downsample_ratio: 3
-    dynamic:
-      enabled: true
-      exclude_latest_local_frames: 4
-      recent_frames: 4
-      conv_kernel_t: 3
-      conv_stride_t: 2
-    revisit:
-      enabled: true
-      deterministic_pose_retrieval: true
-      fov_overlap_threshold: 0.6
-      high_quality_fov_threshold: 0.7
-      plucker_weight: 0.1
-      max_frames: 2
-      fov_half_h: 52.5
-      fov_half_v: 37.5
-      fov_radius: 30.0
-      fov_yaw_samples: 25
-      fov_pitch_samples: 20
-      fov_depth_samples: 20
-      pose_preselect_topk: 64
-      plucker_grid_h: 4
-      plucker_grid_w: 4
-      plucker_focal_length: 0.35
-      compress:
-        downsample_ratio: 3
-    eval_ablation:
-      enabled: false
-      branch: A_plus_D_plus_R_normal
-    generated_history_proxy:
-      enabled: true
-      start_step: 40000
-      ramp_steps: 40000
-      max_prob: 0.25
-      noise_std: 0.25
-      dropout_prob: 0.0
-    injection:
-      dit_hidden_size: 1024
-      anchor_gate: 1.0
-      dynamic_gate: 1.0
-      revisit_gate: 1.0
-    cache:
-      enabled: true
-      device: cpu
-      keep_raw_latents: all
-      keep_compressed_records: true
-      keep_prefix_anchors: true
-      eviction_policy: none
-      no_evict: true
-      clear_between_videos: true
-      max_records: null
-      on_capacity_exceeded: warn
-    checkpoint:
-      strict_dememwm_eval_load: true
-  n_tokens: 4
-debug: false
-wandb:
-  entity: turlin
-  project: worldmem
-  mode: offline
-resume: null
-load: null
-auto_resume: true
-resume_ckpt_path: null
-name: train_dememwm_full_berzelius_8a100_bs16_global128_350k
-output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/
-customized_load: true
-seperate_load: true
-diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-only_tune_memory: false

outputs/2026-06-04/06-24-27/.hydra/hydra.yaml

@@ -1,230 +0,0 @@
-hydra:
-  run:
-    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
-  sweep:
-    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
-    subdir: ${hydra.job.num}
-  launcher:
-    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
-  sweeper:
-    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
-    max_batch_size: null
-    params: null
-  help:
-    app_name: ${hydra.job.name}
-    header: '${hydra.help.app_name} is powered by Hydra.
-
-      '
-    footer: 'Powered by Hydra (https://hydra.cc)
-
-      Use --hydra-help to view Hydra specific help
-
-      '
-    template: '${hydra.help.header}
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (group=option)
-
-
-      $APP_CONFIG_GROUPS
-
-
-      == Config ==
-
-      Override anything in the config (foo.bar=value)
-
-
-      $CONFIG
-
-
-      ${hydra.help.footer}
-
-      '
-  hydra_help:
-    template: 'Hydra (${hydra.runtime.version})
-
-      See https://hydra.cc for more info.
-
-
-      == Flags ==
-
-      $FLAGS_HELP
-
-
-      == Configuration groups ==
-
-      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
-      to command line)
-
-
-      $HYDRA_CONFIG_GROUPS
-
-
-      Use ''--cfg hydra'' to Show the Hydra config.
-
-      '
-    hydra_help: ???
-  hydra_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][HYDRA] %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-    root:
-      level: INFO
-      handlers:
-      - console
-    loggers:
-      logging_example:
-        level: DEBUG
-    disable_existing_loggers: false
-  job_logging:
-    version: 1
-    formatters:
-      simple:
-        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        stream: ext://sys.stdout
-      file:
-        class: logging.FileHandler
-        formatter: simple
-        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
-    root:
-      level: INFO
-      handlers:
-      - console
-      - file
-    disable_existing_loggers: false
-  env: {}
-  mode: RUN
-  searchpath: []
-  callbacks: {}
-  output_subdir: .hydra
-  overrides:
-    hydra:
-    - hydra.mode=RUN
-    task:
-    - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k
-    - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/
-    - auto_resume=true
-    - experiment.tasks=[training]
-    - algorithm=dememwm_memory_dit
-    - +customized_load=true
-    - +seperate_load=true
-    - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-    - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-    - +only_tune_memory=false
-    - dataset=video_minecraft_latent
-    - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-    - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-    - dataset.n_frames=1000
-    - +dataset.n_frames_valid=500
-    - +dataset.customized_validation=true
-    - +dataset.memory_condition_length=0
-    - +dataset.wo_updown=false
-    - +dataset.angle_range=180
-    - +dataset.pos_range=8
-    - ++algorithm.n_tokens=4
-    - algorithm.x_shape=[16,18,32]
-    - ++algorithm.context_frames=100
-    - ++algorithm.log_video=true
-    - ++algorithm.diffusion.sampling_timesteps=20
-    - ++algorithm.dememwm.generated_history_proxy.enabled=true
-    - ++algorithm.dememwm.generated_history_proxy.start_step=40000
-    - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-    - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-    - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-    - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-    - ++algorithm.dememwm.anchor.enabled=true
-    - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-    - ++algorithm.dememwm.anchor.diverse_selection=true
-    - ++algorithm.dememwm.anchor.compress.downsample_ratio=3
-    - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-    - ++algorithm.dememwm.dynamic.enabled=true
-    - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4
-    - ++algorithm.dememwm.dynamic.recent_frames=4
-    - ++algorithm.dememwm.revisit.enabled=true
-    - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-    - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-    - ++algorithm.dememwm.revisit.pose_preselect_topk=64
-    - ++algorithm.dememwm.revisit.fov_yaw_samples=25
-    - ++algorithm.dememwm.revisit.fov_pitch_samples=20
-    - ++algorithm.dememwm.revisit.fov_depth_samples=20
-    - ++algorithm.dememwm.revisit.plucker_weight=0.10
-    - ++algorithm.dememwm.revisit.max_frames=2
-    - ++algorithm.dememwm.revisit.compress.downsample_ratio=3
-    - ++algorithm.dememwm.cache.enabled=true
-    - ++algorithm.dememwm.cache.device=cpu
-    - ++algorithm.dememwm.cache.keep_raw_latents=all
-    - ++algorithm.dememwm.cache.keep_compressed_records=true
-    - ++algorithm.dememwm.cache.eviction_policy=none
-    - ++algorithm.dememwm.cache.no_evict=true
-    - ++algorithm.dememwm.cache.clear_between_videos=true
-    - ++algorithm.dememwm.cache.max_records=null
-    - ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-    - ++algorithm.dememwm.curriculum.enabled=true
-    - ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-    - ++algorithm.dememwm.curriculum.freeze_vae=true
-    - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-    - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-    - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-    - experiment.training.batch_size=16
-    - experiment.training.optim.accumulate_grad_batches=1
-    - experiment.validation.batch_size=1
-    - experiment.validation.limit_batch=1
-    - experiment.training.checkpointing.every_n_train_steps=5000
-    - experiment.validation.val_every_n_step=5000
-    - experiment.training.max_steps=350000
-  job:
-    name: main
-    chdir: null
-    override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000
-    id: ???
-    num: ???
-    config_name: training
-    env_set: {}
-    env_copy: []
-    config:
-      override_dirname:
-        kv_sep: '='
-        item_sep: ','
-        exclude_keys: []
-  runtime:
-    version: 1.3.2
-    version_base: '1.3'
-    cwd: /proj/cvl/users/x_fahkh2/DeMemWM
-    config_sources:
-    - path: hydra.conf
-      schema: pkg
-      provider: hydra
-    - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations
-      schema: file
-      provider: main
-    - path: ''
-      schema: structured
-      provider: schema
-    output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-04/06-24-27
-    choices:
-      cluster: null
-      algorithm: dememwm_memory_dit
-      dataset: video_minecraft_latent
-      experiment: exp_video
-      hydra/env: default
-      hydra/callbacks: null
-      hydra/job_logging: default
-      hydra/hydra_logging: default
-      hydra/hydra_help: default
-      hydra/help: default
-      hydra/sweeper: basic
-      hydra/launcher: basic
-      hydra/output: default
-  verbose: false

outputs/2026-06-04/06-24-27/.hydra/overrides.yaml

@@ -1,72 +0,0 @@
-- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k
-- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/
-- auto_resume=true
-- experiment.tasks=[training]
-- algorithm=dememwm_memory_dit
-- +customized_load=true
-- +seperate_load=true
-- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors
-- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors
-- +only_tune_memory=false
-- dataset=video_minecraft_latent
-- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
-- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
-- dataset.n_frames=1000
-- +dataset.n_frames_valid=500
-- +dataset.customized_validation=true
-- +dataset.memory_condition_length=0
-- +dataset.wo_updown=false
-- +dataset.angle_range=180
-- +dataset.pos_range=8
-- ++algorithm.n_tokens=4
-- algorithm.x_shape=[16,18,32]
-- ++algorithm.context_frames=100
-- ++algorithm.log_video=true
-- ++algorithm.diffusion.sampling_timesteps=20
-- ++algorithm.dememwm.generated_history_proxy.enabled=true
-- ++algorithm.dememwm.generated_history_proxy.start_step=40000
-- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000
-- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25
-- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25
-- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0
-- ++algorithm.dememwm.anchor.enabled=true
-- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3]
-- ++algorithm.dememwm.anchor.diverse_selection=true
-- ++algorithm.dememwm.anchor.compress.downsample_ratio=3
-- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false
-- ++algorithm.dememwm.dynamic.enabled=true
-- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4
-- ++algorithm.dememwm.dynamic.recent_frames=4
-- ++algorithm.dememwm.revisit.enabled=true
-- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true
-- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60
-- ++algorithm.dememwm.revisit.pose_preselect_topk=64
-- ++algorithm.dememwm.revisit.fov_yaw_samples=25
-- ++algorithm.dememwm.revisit.fov_pitch_samples=20
-- ++algorithm.dememwm.revisit.fov_depth_samples=20
-- ++algorithm.dememwm.revisit.plucker_weight=0.10
-- ++algorithm.dememwm.revisit.max_frames=2
-- ++algorithm.dememwm.revisit.compress.downsample_ratio=3
-- ++algorithm.dememwm.cache.enabled=true
-- ++algorithm.dememwm.cache.device=cpu
-- ++algorithm.dememwm.cache.keep_raw_latents=all
-- ++algorithm.dememwm.cache.keep_compressed_records=true
-- ++algorithm.dememwm.cache.eviction_policy=none
-- ++algorithm.dememwm.cache.no_evict=true
-- ++algorithm.dememwm.cache.clear_between_videos=true
-- ++algorithm.dememwm.cache.max_records=null
-- ++algorithm.dememwm.cache.on_capacity_exceeded=warn
-- ++algorithm.dememwm.curriculum.enabled=true
-- ++algorithm.dememwm.curriculum.full_stage_start_step=20000
-- ++algorithm.dememwm.curriculum.freeze_vae=true
-- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true
-- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5
-- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5
-- experiment.training.batch_size=16
-- experiment.training.optim.accumulate_grad_batches=1
-- experiment.validation.batch_size=1
-- experiment.validation.limit_batch=1
-- experiment.training.checkpointing.every_n_train_steps=5000
-- experiment.validation.val_every_n_step=5000
-- experiment.training.max_steps=350000

outputs/2026-06-04/06-24-27/main.log

@@ -1,77 +0,0 @@
-[2026-06-04 06:24:58,261][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP)
-[2026-06-04 06:24:58,374][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True
-[2026-06-04 06:24:58,374][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores
-[2026-06-04 06:24:58,374][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs
-[2026-06-04 06:24:58,374][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs
-[2026-06-04 06:24:58,374][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used.
-[2026-06-04 06:24:58,617][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix=''
-[2026-06-04 06:24:58,844][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix=''
-[2026-06-04 06:25:01,130][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8
-[2026-06-04 06:25:01,209][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8
-[2026-06-04 06:25:01,432][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8
-[2026-06-04 06:25:01,930][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8
-[2026-06-04 06:25:02,129][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8
-[2026-06-04 06:25:02,209][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8
-[2026-06-04 06:25:03,309][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8
-[2026-06-04 06:25:07,333][pytorch_lightning.utilities.rank_zero][INFO] - ----------------------------------------------------------------------------------------------------
-distributed_backend=nccl
-All distributed processes registered. Starting with 8 processes
-----------------------------------------------------------------------------------------------------
-
-[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-04 06:25:20,837][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
-[2026-06-04 06:25:21,907][lightning.pytorch.callbacks.model_summary][INFO] - 
-  | Name                       | Type                                  | Params
--------------------------------------------------------------------------------------
-0 | diffusion_model            | Diffusion                             | 910 M 
-1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor         | 199 K 
-2 | dememwm_anchor_proj        | SpatialConv2DMemoryProjector          | 591 K 
-3 | dememwm_revisit_proj       | SpatialConv2DMemoryProjector          | 591 K 
-4 | dememwm_revisit_gate       | RevisitRawGate                        | 4     
-5 | validation_lpips_model     | LearnedPerceptualImagePatchSimilarity | 2.5 M 
-6 | vae                        | AutoencoderKL                         | 229 M 
--------------------------------------------------------------------------------------
-911 M     Trainable params
-231 M     Non-trainable params
-1.1 B     Total params
-4,573.841 Total estimated model params size (MB)
-[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers.
-[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15
-[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15
-[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15
-[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15
-[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15
-[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15
-[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15
-[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15
-[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15
-[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15
-[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15
-[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15
-[2026-06-07 06:24:48,021][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,021][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15
-[2026-06-07 06:24:48,021][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15
-[2026-06-07 06:24:48,021][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15
-[2026-06-07 06:24:48,021][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15